2024-11-26 14:31:03,065 main DEBUG Apache Log4j Core 2.17.2 initializing configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@20b2475a 2024-11-26 14:31:03,077 main DEBUG Took 0.009627 seconds to load 1 plugins from package org.apache.hadoop.hbase.logging 2024-11-26 14:31:03,077 main DEBUG PluginManager 'Core' found 129 plugins 2024-11-26 14:31:03,078 main DEBUG PluginManager 'Level' found 0 plugins 2024-11-26 14:31:03,079 main DEBUG PluginManager 'Lookup' found 16 plugins 2024-11-26 14:31:03,080 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-26 14:31:03,087 main DEBUG PluginManager 'TypeConverter' found 26 plugins 2024-11-26 14:31:03,099 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.util.MBeans", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-26 14:31:03,100 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-26 14:31:03,101 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.logging.TestJul2Slf4j", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-26 14:31:03,101 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-26 14:31:03,102 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.zookeeper", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-26 14:31:03,102 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-26 14:31:03,103 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSinkAdapter", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-26 14:31:03,103 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-26 14:31:03,104 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSystemImpl", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-26 14:31:03,104 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-26 14:31:03,105 main DEBUG LoggerConfig$Builder(additivity="false", level="WARN", levelAndRefs="null", name="org.apache.directory", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-26 14:31:03,105 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-26 14:31:03,106 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.ipc.FailedServers", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-26 14:31:03,106 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-26 14:31:03,106 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsConfig", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-26 14:31:03,107 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-26 14:31:03,107 main DEBUG LoggerConfig$Builder(additivity="null", level="INFO", levelAndRefs="null", name="org.apache.hadoop.hbase.ScheduledChore", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-26 14:31:03,107 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-26 14:31:03,108 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.regionserver.RSRpcServices", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-26 14:31:03,108 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-26 14:31:03,109 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-26 14:31:03,109 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-26 14:31:03,109 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-26 14:31:03,109 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-26 14:31:03,110 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hbase.thirdparty.io.netty.channel", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-26 14:31:03,110 main DEBUG Building Plugin[name=root, class=org.apache.logging.log4j.core.config.LoggerConfig$RootLogger]. 2024-11-26 14:31:03,112 main DEBUG LoggerConfig$RootLogger$Builder(additivity="null", level="null", levelAndRefs="INFO,Console", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-26 14:31:03,113 main DEBUG Building Plugin[name=loggers, class=org.apache.logging.log4j.core.config.LoggersPlugin]. 2024-11-26 14:31:03,115 main DEBUG createLoggers(={org.apache.hadoop.metrics2.util.MBeans, org.apache.hadoop.hbase.logging.TestJul2Slf4j, org.apache.zookeeper, org.apache.hadoop.metrics2.impl.MetricsSinkAdapter, org.apache.hadoop.metrics2.impl.MetricsSystemImpl, org.apache.directory, org.apache.hadoop.hbase.ipc.FailedServers, org.apache.hadoop.metrics2.impl.MetricsConfig, org.apache.hadoop.hbase.ScheduledChore, org.apache.hadoop.hbase.regionserver.RSRpcServices, org.apache.hadoop, org.apache.hadoop.hbase, org.apache.hbase.thirdparty.io.netty.channel, root}) 2024-11-26 14:31:03,115 main DEBUG Building Plugin[name=layout, class=org.apache.logging.log4j.core.layout.PatternLayout]. 2024-11-26 14:31:03,116 main DEBUG PatternLayout$Builder(pattern="%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n", PatternSelector=null, Configuration(PropertiesConfig), Replace=null, charset="null", alwaysWriteExceptions="null", disableAnsi="null", noConsoleNoAnsi="null", header="null", footer="null") 2024-11-26 14:31:03,117 main DEBUG PluginManager 'Converter' found 47 plugins 2024-11-26 14:31:03,125 main DEBUG Building Plugin[name=appender, class=org.apache.hadoop.hbase.logging.HBaseTestAppender]. 2024-11-26 14:31:03,128 main DEBUG HBaseTestAppender$Builder(target="SYSTEM_ERR", maxSize="1G", bufferedIo="null", bufferSize="null", immediateFlush="null", ignoreExceptions="null", PatternLayout(%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n), name="Console", Configuration(PropertiesConfig), Filter=null, ={}) 2024-11-26 14:31:03,129 main DEBUG Starting HBaseTestOutputStreamManager SYSTEM_ERR 2024-11-26 14:31:03,130 main DEBUG Building Plugin[name=appenders, class=org.apache.logging.log4j.core.config.AppendersPlugin]. 2024-11-26 14:31:03,130 main DEBUG createAppenders(={Console}) 2024-11-26 14:31:03,131 main DEBUG Configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@20b2475a initialized 2024-11-26 14:31:03,131 main DEBUG Starting configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@20b2475a 2024-11-26 14:31:03,132 main DEBUG Started configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@20b2475a OK. 2024-11-26 14:31:03,132 main DEBUG Shutting down OutputStreamManager SYSTEM_OUT.false.false-1 2024-11-26 14:31:03,132 main DEBUG OutputStream closed 2024-11-26 14:31:03,133 main DEBUG Shut down OutputStreamManager SYSTEM_OUT.false.false-1, all resources released: true 2024-11-26 14:31:03,133 main DEBUG Appender DefaultConsole-1 stopped with status true 2024-11-26 14:31:03,133 main DEBUG Stopped org.apache.logging.log4j.core.config.DefaultConfiguration@4310d43 OK 2024-11-26 14:31:03,205 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6 2024-11-26 14:31:03,207 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=StatusLogger 2024-11-26 14:31:03,208 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=ContextSelector 2024-11-26 14:31:03,209 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name= 2024-11-26 14:31:03,210 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.directory 2024-11-26 14:31:03,210 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSinkAdapter 2024-11-26 14:31:03,210 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.zookeeper 2024-11-26 14:31:03,211 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.logging.TestJul2Slf4j 2024-11-26 14:31:03,211 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSystemImpl 2024-11-26 14:31:03,212 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.util.MBeans 2024-11-26 14:31:03,212 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase 2024-11-26 14:31:03,212 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop 2024-11-26 14:31:03,213 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ipc.FailedServers 2024-11-26 14:31:03,213 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.regionserver.RSRpcServices 2024-11-26 14:31:03,213 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsConfig 2024-11-26 14:31:03,214 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hbase.thirdparty.io.netty.channel 2024-11-26 14:31:03,214 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ScheduledChore 2024-11-26 14:31:03,215 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Appenders,name=Console 2024-11-26 14:31:03,217 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-26 14:31:03,217 main DEBUG Reconfiguration complete for context[name=1dbd16a6] at URI jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-logging/target/hbase-logging-3.0.0-beta-2-SNAPSHOT-tests.jar!/log4j2.properties (org.apache.logging.log4j.core.LoggerContext@12f9af83) with optional ClassLoader: null 2024-11-26 14:31:03,217 main DEBUG Shutdown hook enabled. Registering a new one. 2024-11-26 14:31:03,218 main DEBUG LoggerContext[name=1dbd16a6, org.apache.logging.log4j.core.LoggerContext@12f9af83] started OK. 2024-11-26T14:31:03,442 DEBUG [main {}] hbase.HBaseTestingUtil(323): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/175d8e9e-4f50-9d71-fd80-148275c72391 2024-11-26 14:31:03,444 main DEBUG AsyncLogger.ThreadNameStrategy=UNCACHED (user specified null, default is UNCACHED) 2024-11-26 14:31:03,444 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-26T14:31:03,452 INFO [main {}] hbase.HBaseClassTestRule(94): Test class org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay timeout: 13 mins 2024-11-26T14:31:03,458 INFO [main {}] hbase.HBaseClassTestRule(94): Test class org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplayCompressed timeout: 13 mins 2024-11-26T14:31:03,479 DEBUG [Time-limited test {}] channel.MultithreadEventLoopGroup(44): -Dio.netty.eventLoopThreads: 16 2024-11-26T14:31:03,534 DEBUG [Time-limited test {}] nio.NioEventLoop(110): -Dio.netty.noKeySetOptimization: false 2024-11-26T14:31:03,535 DEBUG [Time-limited test {}] nio.NioEventLoop(111): -Dio.netty.selectorAutoRebuildThreshold: 512 2024-11-26T14:31:03,546 INFO [Time-limited test {}] hbase.HBaseTestingUtil(805): Starting up minicluster with option: StartMiniClusterOption{numMasters=1, masterClass=null, numRegionServers=3, rsPorts=, rsClass=null, numDataNodes=3, dataNodeHosts=null, numZkServers=1, createRootDir=false, createWALDir=false} 2024-11-26T14:31:03,562 INFO [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/175d8e9e-4f50-9d71-fd80-148275c72391/cluster_49e351ba-434f-d14a-4712-fb0fcff718bc, deleteOnExit=true 2024-11-26T14:31:03,563 INFO [Time-limited test {}] hbase.HBaseTestingUtil(818): STARTING DFS 2024-11-26T14:31:03,563 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/175d8e9e-4f50-9d71-fd80-148275c72391/test.cache.data in system properties and HBase conf 2024-11-26T14:31:03,564 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/175d8e9e-4f50-9d71-fd80-148275c72391/hadoop.tmp.dir in system properties and HBase conf 2024-11-26T14:31:03,565 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/175d8e9e-4f50-9d71-fd80-148275c72391/hadoop.log.dir in system properties and HBase conf 2024-11-26T14:31:03,565 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/175d8e9e-4f50-9d71-fd80-148275c72391/mapreduce.cluster.local.dir in system properties and HBase conf 2024-11-26T14:31:03,566 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/175d8e9e-4f50-9d71-fd80-148275c72391/mapreduce.cluster.temp.dir in system properties and HBase conf 2024-11-26T14:31:03,566 INFO [Time-limited test {}] hbase.HBaseTestingUtil(738): read short circuit is OFF 2024-11-26T14:31:03,665 WARN [Time-limited test {}] util.NativeCodeLoader(60): Unable to load native-hadoop library for your platform... using builtin-java classes where applicable 2024-11-26T14:31:03,755 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-26T14:31:03,759 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/175d8e9e-4f50-9d71-fd80-148275c72391/yarn.node-labels.fs-store.root-dir in system properties and HBase conf 2024-11-26T14:31:03,760 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/175d8e9e-4f50-9d71-fd80-148275c72391/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf 2024-11-26T14:31:03,760 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/175d8e9e-4f50-9d71-fd80-148275c72391/yarn.nodemanager.log-dirs in system properties and HBase conf 2024-11-26T14:31:03,760 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/175d8e9e-4f50-9d71-fd80-148275c72391/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-11-26T14:31:03,761 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/175d8e9e-4f50-9d71-fd80-148275c72391/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf 2024-11-26T14:31:03,761 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/175d8e9e-4f50-9d71-fd80-148275c72391/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf 2024-11-26T14:31:03,762 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/175d8e9e-4f50-9d71-fd80-148275c72391/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-11-26T14:31:03,762 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/175d8e9e-4f50-9d71-fd80-148275c72391/dfs.journalnode.edits.dir in system properties and HBase conf 2024-11-26T14:31:03,762 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/175d8e9e-4f50-9d71-fd80-148275c72391/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf 2024-11-26T14:31:03,763 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/175d8e9e-4f50-9d71-fd80-148275c72391/nfs.dump.dir in system properties and HBase conf 2024-11-26T14:31:03,763 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/175d8e9e-4f50-9d71-fd80-148275c72391/java.io.tmpdir in system properties and HBase conf 2024-11-26T14:31:03,764 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/175d8e9e-4f50-9d71-fd80-148275c72391/dfs.journalnode.edits.dir in system properties and HBase conf 2024-11-26T14:31:03,764 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/175d8e9e-4f50-9d71-fd80-148275c72391/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf 2024-11-26T14:31:03,764 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/175d8e9e-4f50-9d71-fd80-148275c72391/fs.s3a.committer.staging.tmp.path in system properties and HBase conf 2024-11-26T14:31:04,945 WARN [Time-limited test {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-namenode.properties,hadoop-metrics2.properties 2024-11-26T14:31:05,007 INFO [Time-limited test {}] log.Log(170): Logging initialized @2592ms to org.eclipse.jetty.util.log.Slf4jLog 2024-11-26T14:31:05,067 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-26T14:31:05,123 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-26T14:31:05,150 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-26T14:31:05,150 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-26T14:31:05,152 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-11-26T14:31:05,163 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-26T14:31:05,166 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@6630be9c{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/175d8e9e-4f50-9d71-fd80-148275c72391/hadoop.log.dir/,AVAILABLE} 2024-11-26T14:31:05,167 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@70596a48{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-26T14:31:05,335 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@1843526c{hdfs,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/175d8e9e-4f50-9d71-fd80-148275c72391/java.io.tmpdir/jetty-localhost-46847-hadoop-hdfs-3_4_1-tests_jar-_-any-11919004422379686352/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-11-26T14:31:05,340 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@776c4d96{HTTP/1.1, (http/1.1)}{localhost:46847} 2024-11-26T14:31:05,341 INFO [Time-limited test {}] server.Server(415): Started @2925ms 2024-11-26T14:31:05,893 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-26T14:31:05,903 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-26T14:31:05,904 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-26T14:31:05,905 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-26T14:31:05,905 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-11-26T14:31:05,906 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@4b895d95{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/175d8e9e-4f50-9d71-fd80-148275c72391/hadoop.log.dir/,AVAILABLE} 2024-11-26T14:31:05,907 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@3d33512e{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-26T14:31:06,016 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@4dc44ec6{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/175d8e9e-4f50-9d71-fd80-148275c72391/java.io.tmpdir/jetty-localhost-38939-hadoop-hdfs-3_4_1-tests_jar-_-any-16355819280776007213/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-26T14:31:06,017 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@17f6e478{HTTP/1.1, (http/1.1)}{localhost:38939} 2024-11-26T14:31:06,017 INFO [Time-limited test {}] server.Server(415): Started @3602ms 2024-11-26T14:31:06,065 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-11-26T14:31:06,176 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-26T14:31:06,181 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-26T14:31:06,189 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-26T14:31:06,189 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-26T14:31:06,189 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-11-26T14:31:06,190 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@e971547{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/175d8e9e-4f50-9d71-fd80-148275c72391/hadoop.log.dir/,AVAILABLE} 2024-11-26T14:31:06,191 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@3264f7d{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-26T14:31:06,299 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@28e1ba78{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/175d8e9e-4f50-9d71-fd80-148275c72391/java.io.tmpdir/jetty-localhost-36851-hadoop-hdfs-3_4_1-tests_jar-_-any-2861738761467784441/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-26T14:31:06,300 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@702c0733{HTTP/1.1, (http/1.1)}{localhost:36851} 2024-11-26T14:31:06,300 INFO [Time-limited test {}] server.Server(415): Started @3885ms 2024-11-26T14:31:06,303 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-11-26T14:31:06,341 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-26T14:31:06,348 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-26T14:31:06,349 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-26T14:31:06,350 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-26T14:31:06,350 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2024-11-26T14:31:06,351 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@1bf0fdb4{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/175d8e9e-4f50-9d71-fd80-148275c72391/hadoop.log.dir/,AVAILABLE} 2024-11-26T14:31:06,351 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@6853f5bc{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-26T14:31:06,458 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@4290616c{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/175d8e9e-4f50-9d71-fd80-148275c72391/java.io.tmpdir/jetty-localhost-42995-hadoop-hdfs-3_4_1-tests_jar-_-any-11365624260965956770/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-26T14:31:06,459 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@5e25d2b6{HTTP/1.1, (http/1.1)}{localhost:42995} 2024-11-26T14:31:06,459 INFO [Time-limited test {}] server.Server(415): Started @4044ms 2024-11-26T14:31:06,461 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-11-26T14:31:07,764 WARN [Thread-123 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/175d8e9e-4f50-9d71-fd80-148275c72391/cluster_49e351ba-434f-d14a-4712-fb0fcff718bc/data/data1/current/BP-1347678837-172.17.0.3-1732631464290/current, will proceed with Du for space computation calculation, 2024-11-26T14:31:07,764 WARN [Thread-122 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/175d8e9e-4f50-9d71-fd80-148275c72391/cluster_49e351ba-434f-d14a-4712-fb0fcff718bc/data/data3/current/BP-1347678837-172.17.0.3-1732631464290/current, will proceed with Du for space computation calculation, 2024-11-26T14:31:07,764 WARN [Thread-125 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/175d8e9e-4f50-9d71-fd80-148275c72391/cluster_49e351ba-434f-d14a-4712-fb0fcff718bc/data/data2/current/BP-1347678837-172.17.0.3-1732631464290/current, will proceed with Du for space computation calculation, 2024-11-26T14:31:07,764 WARN [Thread-124 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/175d8e9e-4f50-9d71-fd80-148275c72391/cluster_49e351ba-434f-d14a-4712-fb0fcff718bc/data/data4/current/BP-1347678837-172.17.0.3-1732631464290/current, will proceed with Du for space computation calculation, 2024-11-26T14:31:07,803 WARN [Thread-81 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-11-26T14:31:07,803 WARN [Thread-58 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-11-26T14:31:07,819 WARN [Thread-142 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/175d8e9e-4f50-9d71-fd80-148275c72391/cluster_49e351ba-434f-d14a-4712-fb0fcff718bc/data/data5/current/BP-1347678837-172.17.0.3-1732631464290/current, will proceed with Du for space computation calculation, 2024-11-26T14:31:07,820 WARN [Thread-143 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/175d8e9e-4f50-9d71-fd80-148275c72391/cluster_49e351ba-434f-d14a-4712-fb0fcff718bc/data/data6/current/BP-1347678837-172.17.0.3-1732631464290/current, will proceed with Du for space computation calculation, 2024-11-26T14:31:07,844 WARN [Thread-103 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-11-26T14:31:07,850 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x12e7429f393ab0d with lease ID 0x9a2578ca6cb960d9: Processing first storage report for DS-9cb9383d-a6c8-4ba6-83b3-6706331a6539 from datanode DatanodeRegistration(127.0.0.1:36277, datanodeUuid=dc8cdb2d-b696-4216-9a09-4a5ee6dadb62, infoPort=40449, infoSecurePort=0, ipcPort=37205, storageInfo=lv=-57;cid=testClusterID;nsid=1357334789;c=1732631464290) 2024-11-26T14:31:07,851 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x12e7429f393ab0d with lease ID 0x9a2578ca6cb960d9: from storage DS-9cb9383d-a6c8-4ba6-83b3-6706331a6539 node DatanodeRegistration(127.0.0.1:36277, datanodeUuid=dc8cdb2d-b696-4216-9a09-4a5ee6dadb62, infoPort=40449, infoSecurePort=0, ipcPort=37205, storageInfo=lv=-57;cid=testClusterID;nsid=1357334789;c=1732631464290), blocks: 0, hasStaleStorage: true, processing time: 1 msecs, invalidatedBlocks: 0 2024-11-26T14:31:07,852 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x217a40a509e0cc89 with lease ID 0x9a2578ca6cb960da: Processing first storage report for DS-44b12d44-38d0-4c76-b56f-3ca8d8caa398 from datanode DatanodeRegistration(127.0.0.1:38147, datanodeUuid=cf2dc361-1427-41a9-835e-da08867ae736, infoPort=35081, infoSecurePort=0, ipcPort=36561, storageInfo=lv=-57;cid=testClusterID;nsid=1357334789;c=1732631464290) 2024-11-26T14:31:07,852 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x217a40a509e0cc89 with lease ID 0x9a2578ca6cb960da: from storage DS-44b12d44-38d0-4c76-b56f-3ca8d8caa398 node DatanodeRegistration(127.0.0.1:38147, datanodeUuid=cf2dc361-1427-41a9-835e-da08867ae736, infoPort=35081, infoSecurePort=0, ipcPort=36561, storageInfo=lv=-57;cid=testClusterID;nsid=1357334789;c=1732631464290), blocks: 0, hasStaleStorage: true, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-26T14:31:07,852 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x85afefe656b2ccb4 with lease ID 0x9a2578ca6cb960db: Processing first storage report for DS-dd37de78-b3d5-4d7c-8223-4b15101c21bf from datanode DatanodeRegistration(127.0.0.1:36307, datanodeUuid=d2854781-abc3-4956-9d56-42b6dfcc9e29, infoPort=34431, infoSecurePort=0, ipcPort=45617, storageInfo=lv=-57;cid=testClusterID;nsid=1357334789;c=1732631464290) 2024-11-26T14:31:07,852 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x85afefe656b2ccb4 with lease ID 0x9a2578ca6cb960db: from storage DS-dd37de78-b3d5-4d7c-8223-4b15101c21bf node DatanodeRegistration(127.0.0.1:36307, datanodeUuid=d2854781-abc3-4956-9d56-42b6dfcc9e29, infoPort=34431, infoSecurePort=0, ipcPort=45617, storageInfo=lv=-57;cid=testClusterID;nsid=1357334789;c=1732631464290), blocks: 0, hasStaleStorage: true, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-26T14:31:07,852 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x12e7429f393ab0d with lease ID 0x9a2578ca6cb960d9: Processing first storage report for DS-5f232ac8-6fc6-4382-8559-9a91d91cabdf from datanode DatanodeRegistration(127.0.0.1:36277, datanodeUuid=dc8cdb2d-b696-4216-9a09-4a5ee6dadb62, infoPort=40449, infoSecurePort=0, ipcPort=37205, storageInfo=lv=-57;cid=testClusterID;nsid=1357334789;c=1732631464290) 2024-11-26T14:31:07,852 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x12e7429f393ab0d with lease ID 0x9a2578ca6cb960d9: from storage DS-5f232ac8-6fc6-4382-8559-9a91d91cabdf node DatanodeRegistration(127.0.0.1:36277, datanodeUuid=dc8cdb2d-b696-4216-9a09-4a5ee6dadb62, infoPort=40449, infoSecurePort=0, ipcPort=37205, storageInfo=lv=-57;cid=testClusterID;nsid=1357334789;c=1732631464290), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-26T14:31:07,853 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x217a40a509e0cc89 with lease ID 0x9a2578ca6cb960da: Processing first storage report for DS-899711b5-c9ab-491b-9d2a-8763a1c47560 from datanode DatanodeRegistration(127.0.0.1:38147, datanodeUuid=cf2dc361-1427-41a9-835e-da08867ae736, infoPort=35081, infoSecurePort=0, ipcPort=36561, storageInfo=lv=-57;cid=testClusterID;nsid=1357334789;c=1732631464290) 2024-11-26T14:31:07,853 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x217a40a509e0cc89 with lease ID 0x9a2578ca6cb960da: from storage DS-899711b5-c9ab-491b-9d2a-8763a1c47560 node DatanodeRegistration(127.0.0.1:38147, datanodeUuid=cf2dc361-1427-41a9-835e-da08867ae736, infoPort=35081, infoSecurePort=0, ipcPort=36561, storageInfo=lv=-57;cid=testClusterID;nsid=1357334789;c=1732631464290), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-26T14:31:07,853 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x85afefe656b2ccb4 with lease ID 0x9a2578ca6cb960db: Processing first storage report for DS-79ea76f3-ac24-4219-9df3-ffb6557b2d4a from datanode DatanodeRegistration(127.0.0.1:36307, datanodeUuid=d2854781-abc3-4956-9d56-42b6dfcc9e29, infoPort=34431, infoSecurePort=0, ipcPort=45617, storageInfo=lv=-57;cid=testClusterID;nsid=1357334789;c=1732631464290) 2024-11-26T14:31:07,853 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x85afefe656b2ccb4 with lease ID 0x9a2578ca6cb960db: from storage DS-79ea76f3-ac24-4219-9df3-ffb6557b2d4a node DatanodeRegistration(127.0.0.1:36307, datanodeUuid=d2854781-abc3-4956-9d56-42b6dfcc9e29, infoPort=34431, infoSecurePort=0, ipcPort=45617, storageInfo=lv=-57;cid=testClusterID;nsid=1357334789;c=1732631464290), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-26T14:31:07,885 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(631): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/175d8e9e-4f50-9d71-fd80-148275c72391 2024-11-26T14:31:07,945 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/175d8e9e-4f50-9d71-fd80-148275c72391/cluster_49e351ba-434f-d14a-4712-fb0fcff718bc/zookeeper_0, clientPort=49403, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/175d8e9e-4f50-9d71-fd80-148275c72391/cluster_49e351ba-434f-d14a-4712-fb0fcff718bc/zookeeper_0/version-2, dataDirSize=457 dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/175d8e9e-4f50-9d71-fd80-148275c72391/cluster_49e351ba-434f-d14a-4712-fb0fcff718bc/zookeeper_0/version-2, dataLogSize=457 tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0 2024-11-26T14:31:07,953 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=49403 2024-11-26T14:31:07,973 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-26T14:31:07,977 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-26T14:31:08,159 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38147 is added to blk_1073741825_1001 (size=7) 2024-11-26T14:31:08,160 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36277 is added to blk_1073741825_1001 (size=7) 2024-11-26T14:31:08,160 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36307 is added to blk_1073741825_1001 (size=7) 2024-11-26T14:31:08,575 INFO [Time-limited test {}] util.FSUtils(489): Created version file at hdfs://localhost:44999/user/jenkins/test-data/2de93217-0593-bca0-5c5f-879b6362c76f with version=8 2024-11-26T14:31:08,576 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1139): Setting hbase.fs.tmp.dir to hdfs://localhost:44999/user/jenkins/test-data/2de93217-0593-bca0-5c5f-879b6362c76f/hbase-staging 2024-11-26T14:31:08,820 INFO [Time-limited test {}] client.ConnectionUtils(128): master/19945b63c2b4:0 server-side Connection retries=45 2024-11-26T14:31:08,827 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-26T14:31:08,828 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-26T14:31:08,832 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-26T14:31:08,832 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-26T14:31:08,832 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-26T14:31:08,948 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService 2024-11-26T14:31:08,998 INFO [Time-limited test {}] metrics.MetricRegistriesLoader(60): Loaded MetricRegistries class org.apache.hadoop.hbase.metrics.impl.MetricRegistriesImpl 2024-11-26T14:31:09,005 DEBUG [Time-limited test {}] util.ClassSize(228): Using Unsafe to estimate memory layout 2024-11-26T14:31:09,008 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-26T14:31:09,030 DEBUG [Time-limited test {}] channel.DefaultChannelId(84): -Dio.netty.processId: 80187 (auto-detected) 2024-11-26T14:31:09,031 DEBUG [Time-limited test {}] channel.DefaultChannelId(106): -Dio.netty.machineId: 02:42:ac:ff:fe:11:00:03 (auto-detected) 2024-11-26T14:31:09,048 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.3:43577 2024-11-26T14:31:09,066 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:43577 connecting to ZooKeeper ensemble=127.0.0.1:49403 2024-11-26T14:31:09,181 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:435770x0, quorum=127.0.0.1:49403, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-26T14:31:09,183 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:43577-0x101778e13450000 connected 2024-11-26T14:31:09,274 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-26T14:31:09,282 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-26T14:31:09,293 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:43577-0x101778e13450000, quorum=127.0.0.1:49403, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-26T14:31:09,297 INFO [Time-limited test {}] master.HMaster(525): hbase.rootdir=hdfs://localhost:44999/user/jenkins/test-data/2de93217-0593-bca0-5c5f-879b6362c76f, hbase.cluster.distributed=false 2024-11-26T14:31:09,319 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:43577-0x101778e13450000, quorum=127.0.0.1:49403, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-26T14:31:09,323 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=43577 2024-11-26T14:31:09,324 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=43577 2024-11-26T14:31:09,324 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=43577 2024-11-26T14:31:09,325 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=43577 2024-11-26T14:31:09,325 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=43577 2024-11-26T14:31:09,415 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/19945b63c2b4:0 server-side Connection retries=45 2024-11-26T14:31:09,416 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-26T14:31:09,416 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-26T14:31:09,416 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-26T14:31:09,417 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-26T14:31:09,417 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-26T14:31:09,419 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-26T14:31:09,422 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-26T14:31:09,422 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.3:35419 2024-11-26T14:31:09,424 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:35419 connecting to ZooKeeper ensemble=127.0.0.1:49403 2024-11-26T14:31:09,425 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-26T14:31:09,427 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-26T14:31:09,458 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:354190x0, quorum=127.0.0.1:49403, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-26T14:31:09,459 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:35419-0x101778e13450001 connected 2024-11-26T14:31:09,459 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:35419-0x101778e13450001, quorum=127.0.0.1:49403, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-26T14:31:09,464 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-26T14:31:09,471 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-26T14:31:09,473 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:35419-0x101778e13450001, quorum=127.0.0.1:49403, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-26T14:31:09,478 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:35419-0x101778e13450001, quorum=127.0.0.1:49403, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-26T14:31:09,479 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=35419 2024-11-26T14:31:09,479 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=35419 2024-11-26T14:31:09,479 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=35419 2024-11-26T14:31:09,480 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=35419 2024-11-26T14:31:09,480 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=35419 2024-11-26T14:31:09,496 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/19945b63c2b4:0 server-side Connection retries=45 2024-11-26T14:31:09,496 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-26T14:31:09,497 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-26T14:31:09,497 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-26T14:31:09,497 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-26T14:31:09,497 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-26T14:31:09,498 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-26T14:31:09,498 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-26T14:31:09,499 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.3:40179 2024-11-26T14:31:09,500 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:40179 connecting to ZooKeeper ensemble=127.0.0.1:49403 2024-11-26T14:31:09,501 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-26T14:31:09,504 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-26T14:31:09,514 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:401790x0, quorum=127.0.0.1:49403, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-26T14:31:09,515 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:401790x0, quorum=127.0.0.1:49403, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-26T14:31:09,515 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:40179-0x101778e13450002 connected 2024-11-26T14:31:09,516 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-26T14:31:09,517 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-26T14:31:09,518 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:40179-0x101778e13450002, quorum=127.0.0.1:49403, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-26T14:31:09,521 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:40179-0x101778e13450002, quorum=127.0.0.1:49403, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-26T14:31:09,522 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=40179 2024-11-26T14:31:09,522 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=40179 2024-11-26T14:31:09,522 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=40179 2024-11-26T14:31:09,523 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=40179 2024-11-26T14:31:09,523 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=40179 2024-11-26T14:31:09,538 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/19945b63c2b4:0 server-side Connection retries=45 2024-11-26T14:31:09,539 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-26T14:31:09,539 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-26T14:31:09,539 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-26T14:31:09,539 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-26T14:31:09,539 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-26T14:31:09,539 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-26T14:31:09,540 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-26T14:31:09,540 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.3:37467 2024-11-26T14:31:09,542 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:37467 connecting to ZooKeeper ensemble=127.0.0.1:49403 2024-11-26T14:31:09,543 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-26T14:31:09,545 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-26T14:31:09,556 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:374670x0, quorum=127.0.0.1:49403, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-26T14:31:09,557 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:37467-0x101778e13450003 connected 2024-11-26T14:31:09,557 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:37467-0x101778e13450003, quorum=127.0.0.1:49403, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-26T14:31:09,558 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-26T14:31:09,559 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-26T14:31:09,560 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:37467-0x101778e13450003, quorum=127.0.0.1:49403, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-26T14:31:09,562 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:37467-0x101778e13450003, quorum=127.0.0.1:49403, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-26T14:31:09,563 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=37467 2024-11-26T14:31:09,564 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=37467 2024-11-26T14:31:09,565 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=37467 2024-11-26T14:31:09,566 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=37467 2024-11-26T14:31:09,566 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=37467 2024-11-26T14:31:09,583 DEBUG [M:0;19945b63c2b4:43577 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;19945b63c2b4:43577 2024-11-26T14:31:09,585 INFO [master/19945b63c2b4:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/19945b63c2b4,43577,1732631468672 2024-11-26T14:31:09,599 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37467-0x101778e13450003, quorum=127.0.0.1:49403, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-26T14:31:09,599 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43577-0x101778e13450000, quorum=127.0.0.1:49403, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-26T14:31:09,599 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35419-0x101778e13450001, quorum=127.0.0.1:49403, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-26T14:31:09,599 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40179-0x101778e13450002, quorum=127.0.0.1:49403, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-26T14:31:09,601 DEBUG [master/19945b63c2b4:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:43577-0x101778e13450000, quorum=127.0.0.1:49403, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/19945b63c2b4,43577,1732631468672 2024-11-26T14:31:09,637 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40179-0x101778e13450002, quorum=127.0.0.1:49403, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-26T14:31:09,637 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43577-0x101778e13450000, quorum=127.0.0.1:49403, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-26T14:31:09,637 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37467-0x101778e13450003, quorum=127.0.0.1:49403, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-26T14:31:09,637 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37467-0x101778e13450003, quorum=127.0.0.1:49403, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-26T14:31:09,637 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40179-0x101778e13450002, quorum=127.0.0.1:49403, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-26T14:31:09,637 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35419-0x101778e13450001, quorum=127.0.0.1:49403, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-26T14:31:09,637 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35419-0x101778e13450001, quorum=127.0.0.1:49403, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-26T14:31:09,638 DEBUG [master/19945b63c2b4:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:43577-0x101778e13450000, quorum=127.0.0.1:49403, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-11-26T14:31:09,639 INFO [master/19945b63c2b4:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/19945b63c2b4,43577,1732631468672 from backup master directory 2024-11-26T14:31:09,651 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43577-0x101778e13450000, quorum=127.0.0.1:49403, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/19945b63c2b4,43577,1732631468672 2024-11-26T14:31:09,651 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35419-0x101778e13450001, quorum=127.0.0.1:49403, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-26T14:31:09,651 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40179-0x101778e13450002, quorum=127.0.0.1:49403, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-26T14:31:09,651 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37467-0x101778e13450003, quorum=127.0.0.1:49403, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-26T14:31:09,651 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43577-0x101778e13450000, quorum=127.0.0.1:49403, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-26T14:31:09,653 WARN [master/19945b63c2b4:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-26T14:31:09,653 INFO [master/19945b63c2b4:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=19945b63c2b4,43577,1732631468672 2024-11-26T14:31:09,656 INFO [master/19945b63c2b4:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating data MemStoreChunkPool with chunk size 2 MB, max count 396, initial count 0 2024-11-26T14:31:09,658 INFO [master/19945b63c2b4:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating index MemStoreChunkPool with chunk size 204.80 KB, max count 440, initial count 0 2024-11-26T14:31:09,710 DEBUG [master/19945b63c2b4:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [hdfs://localhost:44999/user/jenkins/test-data/2de93217-0593-bca0-5c5f-879b6362c76f/hbase.id] with ID: 85e6cd0d-a40d-403d-9d6e-04753ee5ca5e 2024-11-26T14:31:09,710 DEBUG [master/19945b63c2b4:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: hdfs://localhost:44999/user/jenkins/test-data/2de93217-0593-bca0-5c5f-879b6362c76f/.tmp/hbase.id 2024-11-26T14:31:09,724 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38147 is added to blk_1073741826_1002 (size=42) 2024-11-26T14:31:09,725 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36307 is added to blk_1073741826_1002 (size=42) 2024-11-26T14:31:09,725 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36277 is added to blk_1073741826_1002 (size=42) 2024-11-26T14:31:09,726 DEBUG [master/19945b63c2b4:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [hdfs://localhost:44999/user/jenkins/test-data/2de93217-0593-bca0-5c5f-879b6362c76f/.tmp/hbase.id]:[hdfs://localhost:44999/user/jenkins/test-data/2de93217-0593-bca0-5c5f-879b6362c76f/hbase.id] 2024-11-26T14:31:09,769 INFO [master/19945b63c2b4:0:becomeActiveMaster {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-26T14:31:09,773 INFO [master/19945b63c2b4:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem. 2024-11-26T14:31:09,790 INFO [master/19945b63c2b4:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 16ms. 2024-11-26T14:31:09,820 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43577-0x101778e13450000, quorum=127.0.0.1:49403, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-26T14:31:09,820 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40179-0x101778e13450002, quorum=127.0.0.1:49403, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-26T14:31:09,820 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37467-0x101778e13450003, quorum=127.0.0.1:49403, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-26T14:31:09,820 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35419-0x101778e13450001, quorum=127.0.0.1:49403, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-26T14:31:09,844 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38147 is added to blk_1073741827_1003 (size=196) 2024-11-26T14:31:09,845 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36277 is added to blk_1073741827_1003 (size=196) 2024-11-26T14:31:09,845 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36307 is added to blk_1073741827_1003 (size=196) 2024-11-26T14:31:09,858 INFO [master/19945b63c2b4:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-26T14:31:09,859 INFO [master/19945b63c2b4:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2024-11-26T14:31:09,864 INFO [master/19945b63c2b4:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-26T14:31:09,893 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36307 is added to blk_1073741828_1004 (size=1189) 2024-11-26T14:31:09,893 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36277 is added to blk_1073741828_1004 (size=1189) 2024-11-26T14:31:09,894 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38147 is added to blk_1073741828_1004 (size=1189) 2024-11-26T14:31:09,910 INFO [master/19945b63c2b4:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir hdfs://localhost:44999/user/jenkins/test-data/2de93217-0593-bca0-5c5f-879b6362c76f/MasterData/data/master/store 2024-11-26T14:31:09,931 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38147 is added to blk_1073741829_1005 (size=34) 2024-11-26T14:31:09,931 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36307 is added to blk_1073741829_1005 (size=34) 2024-11-26T14:31:09,932 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36277 is added to blk_1073741829_1005 (size=34) 2024-11-26T14:31:09,937 INFO [master/19945b63c2b4:0:becomeActiveMaster {}] throttle.StoreHotnessProtector(112): StoreHotnessProtector is disabled. Set hbase.region.store.parallel.put.limit > 0 to enable, which may help mitigate load under heavy write pressure. 2024-11-26T14:31:09,940 DEBUG [master/19945b63c2b4:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-26T14:31:09,942 DEBUG [master/19945b63c2b4:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-26T14:31:09,942 INFO [master/19945b63c2b4:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-26T14:31:09,942 DEBUG [master/19945b63c2b4:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-26T14:31:09,943 DEBUG [master/19945b63c2b4:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-26T14:31:09,943 DEBUG [master/19945b63c2b4:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-26T14:31:09,943 INFO [master/19945b63c2b4:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-26T14:31:09,944 DEBUG [master/19945b63c2b4:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1732631469941Disabling compacts and flushes for region at 1732631469942 (+1 ms)Disabling writes for close at 1732631469943 (+1 ms)Writing region close event to WAL at 1732631469943Closed at 1732631469943 2024-11-26T14:31:09,947 WARN [master/19945b63c2b4:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: hdfs://localhost:44999/user/jenkins/test-data/2de93217-0593-bca0-5c5f-879b6362c76f/MasterData/data/master/store/.initializing 2024-11-26T14:31:09,947 DEBUG [master/19945b63c2b4:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=hdfs://localhost:44999/user/jenkins/test-data/2de93217-0593-bca0-5c5f-879b6362c76f/MasterData/WALs/19945b63c2b4,43577,1732631468672 2024-11-26T14:31:09,954 INFO [master/19945b63c2b4:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-26T14:31:09,968 INFO [master/19945b63c2b4:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=19945b63c2b4%2C43577%2C1732631468672, suffix=, logDir=hdfs://localhost:44999/user/jenkins/test-data/2de93217-0593-bca0-5c5f-879b6362c76f/MasterData/WALs/19945b63c2b4,43577,1732631468672, archiveDir=hdfs://localhost:44999/user/jenkins/test-data/2de93217-0593-bca0-5c5f-879b6362c76f/MasterData/oldWALs, maxLogs=10 2024-11-26T14:31:09,994 DEBUG [master/19945b63c2b4:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/2de93217-0593-bca0-5c5f-879b6362c76f/MasterData/WALs/19945b63c2b4,43577,1732631468672/19945b63c2b4%2C43577%2C1732631468672.1732631469972, exclude list is [], retry=0 2024-11-26T14:31:10,010 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(244): No decryptEncryptedDataEncryptionKey method in DFSClient, should be hadoop version with HDFS-12396 java.lang.NoSuchMethodException: org.apache.hadoop.hdfs.DFSClient.decryptEncryptedDataEncryptionKey(org.apache.hadoop.fs.FileEncryptionInfo) at java.lang.Class.getDeclaredMethod(Class.java:2675) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelperWithoutHDFS12396(FanOutOneBlockAsyncDFSOutputSaslHelper.java:183) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelper(FanOutOneBlockAsyncDFSOutputSaslHelper.java:242) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.(FanOutOneBlockAsyncDFSOutputSaslHelper.java:253) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.initialize(FanOutOneBlockAsyncDFSOutputHelper.java:413) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper$5.operationComplete(FanOutOneBlockAsyncDFSOutputHelper.java:472) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper$5.operationComplete(FanOutOneBlockAsyncDFSOutputHelper.java:467) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.NettyFutureUtils.lambda$addListener$0(NettyFutureUtils.java:56) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setSuccess0(DefaultPromise.java:625) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.trySuccess(DefaultPromise.java:105) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPromise.trySuccess(DefaultChannelPromise.java:84) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.fulfillConnectPromise(AbstractEpollChannel.java:658) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:696) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:491) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T14:31:10,014 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36277,DS-9cb9383d-a6c8-4ba6-83b3-6706331a6539,DISK] 2024-11-26T14:31:10,014 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38147,DS-44b12d44-38d0-4c76-b56f-3ca8d8caa398,DISK] 2024-11-26T14:31:10,014 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36307,DS-dd37de78-b3d5-4d7c-8223-4b15101c21bf,DISK] 2024-11-26T14:31:10,019 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.ProtobufDecoder(117): Hadoop 3.3 and above shades protobuf. 2024-11-26T14:31:10,055 INFO [master/19945b63c2b4:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/2de93217-0593-bca0-5c5f-879b6362c76f/MasterData/WALs/19945b63c2b4,43577,1732631468672/19945b63c2b4%2C43577%2C1732631468672.1732631469972 2024-11-26T14:31:10,056 DEBUG [master/19945b63c2b4:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:34431:34431),(127.0.0.1/127.0.0.1:40449:40449),(127.0.0.1/127.0.0.1:35081:35081)] 2024-11-26T14:31:10,057 DEBUG [master/19945b63c2b4:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2024-11-26T14:31:10,057 DEBUG [master/19945b63c2b4:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-26T14:31:10,060 DEBUG [master/19945b63c2b4:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2024-11-26T14:31:10,061 DEBUG [master/19945b63c2b4:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2024-11-26T14:31:10,094 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2024-11-26T14:31:10,118 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2024-11-26T14:31:10,121 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:31:10,124 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-26T14:31:10,124 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2024-11-26T14:31:10,128 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2024-11-26T14:31:10,128 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:31:10,129 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-26T14:31:10,129 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2024-11-26T14:31:10,132 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2024-11-26T14:31:10,132 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:31:10,133 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-26T14:31:10,133 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2024-11-26T14:31:10,136 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2024-11-26T14:31:10,136 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:31:10,137 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-26T14:31:10,138 DEBUG [master/19945b63c2b4:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682 2024-11-26T14:31:10,141 DEBUG [master/19945b63c2b4:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44999/user/jenkins/test-data/2de93217-0593-bca0-5c5f-879b6362c76f/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-26T14:31:10,142 DEBUG [master/19945b63c2b4:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44999/user/jenkins/test-data/2de93217-0593-bca0-5c5f-879b6362c76f/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-26T14:31:10,147 DEBUG [master/19945b63c2b4:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682 2024-11-26T14:31:10,148 DEBUG [master/19945b63c2b4:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682 2024-11-26T14:31:10,151 DEBUG [master/19945b63c2b4:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-26T14:31:10,154 DEBUG [master/19945b63c2b4:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2024-11-26T14:31:10,158 DEBUG [master/19945b63c2b4:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44999/user/jenkins/test-data/2de93217-0593-bca0-5c5f-879b6362c76f/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-26T14:31:10,159 INFO [master/19945b63c2b4:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=69955426, jitterRate=0.04241707921028137}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-26T14:31:10,166 DEBUG [master/19945b63c2b4:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682: Writing region info on filesystem at 1732631470072Initializing all the Stores at 1732631470074 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732631470074Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732631470075 (+1 ms)Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732631470075Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732631470075Cleaning up temporary data from old regions at 1732631470148 (+73 ms)Region opened successfully at 1732631470166 (+18 ms) 2024-11-26T14:31:10,167 INFO [master/19945b63c2b4:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2024-11-26T14:31:10,197 DEBUG [master/19945b63c2b4:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@463513f6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=19945b63c2b4/172.17.0.3:0 2024-11-26T14:31:10,223 INFO [master/19945b63c2b4:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating... 2024-11-26T14:31:10,232 INFO [master/19945b63c2b4:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5 2024-11-26T14:31:10,232 INFO [master/19945b63c2b4:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50 2024-11-26T14:31:10,234 INFO [master/19945b63c2b4:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-26T14:31:10,235 INFO [master/19945b63c2b4:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 1 msec 2024-11-26T14:31:10,240 INFO [master/19945b63c2b4:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 4 msec 2024-11-26T14:31:10,240 INFO [master/19945b63c2b4:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2024-11-26T14:31:10,262 INFO [master/19945b63c2b4:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2024-11-26T14:31:10,269 DEBUG [master/19945b63c2b4:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:43577-0x101778e13450000, quorum=127.0.0.1:49403, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error) 2024-11-26T14:31:10,321 DEBUG [master/19945b63c2b4:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false 2024-11-26T14:31:10,324 INFO [master/19945b63c2b4:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2024-11-26T14:31:10,327 DEBUG [master/19945b63c2b4:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:43577-0x101778e13450000, quorum=127.0.0.1:49403, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error) 2024-11-26T14:31:10,335 DEBUG [master/19945b63c2b4:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false 2024-11-26T14:31:10,338 INFO [master/19945b63c2b4:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2024-11-26T14:31:10,342 DEBUG [master/19945b63c2b4:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:43577-0x101778e13450000, quorum=127.0.0.1:49403, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error) 2024-11-26T14:31:10,352 DEBUG [master/19945b63c2b4:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false 2024-11-26T14:31:10,354 DEBUG [master/19945b63c2b4:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:43577-0x101778e13450000, quorum=127.0.0.1:49403, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error) 2024-11-26T14:31:10,367 DEBUG [master/19945b63c2b4:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false 2024-11-26T14:31:10,387 DEBUG [master/19945b63c2b4:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:43577-0x101778e13450000, quorum=127.0.0.1:49403, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error) 2024-11-26T14:31:10,398 DEBUG [master/19945b63c2b4:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false 2024-11-26T14:31:10,409 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40179-0x101778e13450002, quorum=127.0.0.1:49403, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-26T14:31:10,409 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43577-0x101778e13450000, quorum=127.0.0.1:49403, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-26T14:31:10,409 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37467-0x101778e13450003, quorum=127.0.0.1:49403, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-26T14:31:10,409 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35419-0x101778e13450001, quorum=127.0.0.1:49403, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-26T14:31:10,409 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40179-0x101778e13450002, quorum=127.0.0.1:49403, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-26T14:31:10,410 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37467-0x101778e13450003, quorum=127.0.0.1:49403, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-26T14:31:10,410 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35419-0x101778e13450001, quorum=127.0.0.1:49403, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-26T14:31:10,410 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43577-0x101778e13450000, quorum=127.0.0.1:49403, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-26T14:31:10,414 INFO [master/19945b63c2b4:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=19945b63c2b4,43577,1732631468672, sessionid=0x101778e13450000, setting cluster-up flag (Was=false) 2024-11-26T14:31:10,447 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35419-0x101778e13450001, quorum=127.0.0.1:49403, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-26T14:31:10,447 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37467-0x101778e13450003, quorum=127.0.0.1:49403, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-26T14:31:10,447 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43577-0x101778e13450000, quorum=127.0.0.1:49403, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-26T14:31:10,447 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40179-0x101778e13450002, quorum=127.0.0.1:49403, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-26T14:31:10,483 DEBUG [master/19945b63c2b4:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort 2024-11-26T14:31:10,488 DEBUG [master/19945b63c2b4:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=19945b63c2b4,43577,1732631468672 2024-11-26T14:31:10,515 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37467-0x101778e13450003, quorum=127.0.0.1:49403, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-26T14:31:10,515 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35419-0x101778e13450001, quorum=127.0.0.1:49403, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-26T14:31:10,515 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40179-0x101778e13450002, quorum=127.0.0.1:49403, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-26T14:31:10,515 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43577-0x101778e13450000, quorum=127.0.0.1:49403, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-26T14:31:10,546 DEBUG [master/19945b63c2b4:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort 2024-11-26T14:31:10,549 DEBUG [master/19945b63c2b4:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=19945b63c2b4,43577,1732631468672 2024-11-26T14:31:10,557 INFO [master/19945b63c2b4:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at hdfs://localhost:44999/user/jenkins/test-data/2de93217-0593-bca0-5c5f-879b6362c76f/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again 2024-11-26T14:31:10,571 INFO [RS:0;19945b63c2b4:35419 {}] regionserver.HRegionServer(746): ClusterId : 85e6cd0d-a40d-403d-9d6e-04753ee5ca5e 2024-11-26T14:31:10,571 INFO [RS:2;19945b63c2b4:37467 {}] regionserver.HRegionServer(746): ClusterId : 85e6cd0d-a40d-403d-9d6e-04753ee5ca5e 2024-11-26T14:31:10,571 INFO [RS:1;19945b63c2b4:40179 {}] regionserver.HRegionServer(746): ClusterId : 85e6cd0d-a40d-403d-9d6e-04753ee5ca5e 2024-11-26T14:31:10,574 DEBUG [RS:2;19945b63c2b4:37467 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-26T14:31:10,574 DEBUG [RS:1;19945b63c2b4:40179 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-26T14:31:10,574 DEBUG [RS:0;19945b63c2b4:35419 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-26T14:31:10,596 DEBUG [RS:2;19945b63c2b4:37467 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-26T14:31:10,596 DEBUG [RS:1;19945b63c2b4:40179 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-26T14:31:10,596 DEBUG [RS:0;19945b63c2b4:35419 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-26T14:31:10,596 DEBUG [RS:2;19945b63c2b4:37467 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-26T14:31:10,596 DEBUG [RS:0;19945b63c2b4:35419 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-26T14:31:10,596 DEBUG [RS:1;19945b63c2b4:40179 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-26T14:31:10,610 DEBUG [RS:0;19945b63c2b4:35419 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-26T14:31:10,610 DEBUG [RS:2;19945b63c2b4:37467 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-26T14:31:10,610 DEBUG [RS:1;19945b63c2b4:40179 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-26T14:31:10,611 DEBUG [RS:2;19945b63c2b4:37467 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@f30a69b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=19945b63c2b4/172.17.0.3:0 2024-11-26T14:31:10,611 DEBUG [RS:0;19945b63c2b4:35419 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@d2faf67, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=19945b63c2b4/172.17.0.3:0 2024-11-26T14:31:10,611 DEBUG [RS:1;19945b63c2b4:40179 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5f65a29a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=19945b63c2b4/172.17.0.3:0 2024-11-26T14:31:10,625 DEBUG [RS:1;19945b63c2b4:40179 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:1;19945b63c2b4:40179 2024-11-26T14:31:10,625 DEBUG [RS:0;19945b63c2b4:35419 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;19945b63c2b4:35419 2024-11-26T14:31:10,627 DEBUG [RS:2;19945b63c2b4:37467 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:2;19945b63c2b4:37467 2024-11-26T14:31:10,628 INFO [RS:1;19945b63c2b4:40179 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-26T14:31:10,628 INFO [RS:2;19945b63c2b4:37467 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-26T14:31:10,628 INFO [RS:0;19945b63c2b4:35419 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-26T14:31:10,628 INFO [RS:1;19945b63c2b4:40179 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-26T14:31:10,628 INFO [RS:2;19945b63c2b4:37467 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-26T14:31:10,628 INFO [RS:0;19945b63c2b4:35419 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-26T14:31:10,628 DEBUG [RS:1;19945b63c2b4:40179 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-26T14:31:10,628 DEBUG [RS:2;19945b63c2b4:37467 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-26T14:31:10,628 DEBUG [RS:0;19945b63c2b4:35419 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-26T14:31:10,630 INFO [RS:0;19945b63c2b4:35419 {}] regionserver.HRegionServer(2659): reportForDuty to master=19945b63c2b4,43577,1732631468672 with port=35419, startcode=1732631469384 2024-11-26T14:31:10,630 INFO [RS:2;19945b63c2b4:37467 {}] regionserver.HRegionServer(2659): reportForDuty to master=19945b63c2b4,43577,1732631468672 with port=37467, startcode=1732631469538 2024-11-26T14:31:10,630 INFO [RS:1;19945b63c2b4:40179 {}] regionserver.HRegionServer(2659): reportForDuty to master=19945b63c2b4,43577,1732631468672 with port=40179, startcode=1732631469496 2024-11-26T14:31:10,630 DEBUG [master/19945b63c2b4:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-26T14:31:10,641 INFO [master/19945b63c2b4:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2 2024-11-26T14:31:10,643 DEBUG [RS:1;19945b63c2b4:40179 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-26T14:31:10,643 DEBUG [RS:2;19945b63c2b4:37467 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-26T14:31:10,643 DEBUG [RS:0;19945b63c2b4:35419 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-26T14:31:10,648 INFO [master/19945b63c2b4:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc. 2024-11-26T14:31:10,654 DEBUG [master/19945b63c2b4:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: 19945b63c2b4,43577,1732631468672 Number of backup masters: 0 Number of live region servers: 0 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2024-11-26T14:31:10,661 DEBUG [master/19945b63c2b4:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/19945b63c2b4:0, corePoolSize=5, maxPoolSize=5 2024-11-26T14:31:10,661 DEBUG [master/19945b63c2b4:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/19945b63c2b4:0, corePoolSize=5, maxPoolSize=5 2024-11-26T14:31:10,661 DEBUG [master/19945b63c2b4:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/19945b63c2b4:0, corePoolSize=5, maxPoolSize=5 2024-11-26T14:31:10,662 DEBUG [master/19945b63c2b4:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/19945b63c2b4:0, corePoolSize=5, maxPoolSize=5 2024-11-26T14:31:10,662 DEBUG [master/19945b63c2b4:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/19945b63c2b4:0, corePoolSize=10, maxPoolSize=10 2024-11-26T14:31:10,662 DEBUG [master/19945b63c2b4:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/19945b63c2b4:0, corePoolSize=1, maxPoolSize=1 2024-11-26T14:31:10,662 DEBUG [master/19945b63c2b4:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/19945b63c2b4:0, corePoolSize=2, maxPoolSize=2 2024-11-26T14:31:10,662 DEBUG [master/19945b63c2b4:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/19945b63c2b4:0, corePoolSize=1, maxPoolSize=1 2024-11-26T14:31:10,670 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-26T14:31:10,671 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region 2024-11-26T14:31:10,673 INFO [master/19945b63c2b4:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732631500673 2024-11-26T14:31:10,676 INFO [HMaster-EventLoopGroup-2-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:37665, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins.hfs.1 (auth:SIMPLE), service=RegionServerStatusService 2024-11-26T14:31:10,676 INFO [HMaster-EventLoopGroup-2-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:39421, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins.hfs.2 (auth:SIMPLE), service=RegionServerStatusService 2024-11-26T14:31:10,676 INFO [HMaster-EventLoopGroup-2-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:55243, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=RegionServerStatusService 2024-11-26T14:31:10,676 INFO [master/19945b63c2b4:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2024-11-26T14:31:10,677 INFO [master/19945b63c2b4:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2024-11-26T14:31:10,678 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:31:10,678 INFO [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2024-11-26T14:31:10,680 INFO [master/19945b63c2b4:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2024-11-26T14:31:10,681 INFO [master/19945b63c2b4:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2024-11-26T14:31:10,681 INFO [master/19945b63c2b4:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2024-11-26T14:31:10,681 INFO [master/19945b63c2b4:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2024-11-26T14:31:10,682 INFO [master/19945b63c2b4:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-26T14:31:10,684 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=43577 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerStartup(MasterRpcServices.java:667) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16714) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:31:10,685 INFO [master/19945b63c2b4:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2024-11-26T14:31:10,687 INFO [master/19945b63c2b4:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2024-11-26T14:31:10,687 INFO [master/19945b63c2b4:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2024-11-26T14:31:10,709 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=43577 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerStartup(MasterRpcServices.java:667) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16714) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:31:10,710 INFO [master/19945b63c2b4:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2024-11-26T14:31:10,710 INFO [master/19945b63c2b4:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2024-11-26T14:31:10,710 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=43577 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerStartup(MasterRpcServices.java:667) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16714) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T14:31:10,711 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36307 is added to blk_1073741831_1007 (size=1321) 2024-11-26T14:31:10,712 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38147 is added to blk_1073741831_1007 (size=1321) 2024-11-26T14:31:10,713 DEBUG [master/19945b63c2b4:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/19945b63c2b4:0:becomeActiveMaster-HFileCleaner.large.0-1732631470712,5,FailOnTimeoutGroup] 2024-11-26T14:31:10,713 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36277 is added to blk_1073741831_1007 (size=1321) 2024-11-26T14:31:10,713 DEBUG [master/19945b63c2b4:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/19945b63c2b4:0:becomeActiveMaster-HFileCleaner.small.0-1732631470713,5,FailOnTimeoutGroup] 2024-11-26T14:31:10,713 INFO [master/19945b63c2b4:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-26T14:31:10,714 INFO [master/19945b63c2b4:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2024-11-26T14:31:10,715 INFO [master/19945b63c2b4:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2024-11-26T14:31:10,715 INFO [master/19945b63c2b4:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2024-11-26T14:31:10,716 INFO [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to hdfs://localhost:44999/user/jenkins/test-data/2de93217-0593-bca0-5c5f-879b6362c76f/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321 2024-11-26T14:31:10,716 INFO [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:44999/user/jenkins/test-data/2de93217-0593-bca0-5c5f-879b6362c76f 2024-11-26T14:31:10,728 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36277 is added to blk_1073741832_1008 (size=32) 2024-11-26T14:31:10,728 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36307 is added to blk_1073741832_1008 (size=32) 2024-11-26T14:31:10,729 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38147 is added to blk_1073741832_1008 (size=32) 2024-11-26T14:31:10,731 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-26T14:31:10,733 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-26T14:31:10,736 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-26T14:31:10,736 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:31:10,737 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-26T14:31:10,737 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-26T14:31:10,739 DEBUG [RS:0;19945b63c2b4:35419 {}] regionserver.HRegionServer(2683): Master is not running yet 2024-11-26T14:31:10,739 DEBUG [RS:1;19945b63c2b4:40179 {}] regionserver.HRegionServer(2683): Master is not running yet 2024-11-26T14:31:10,739 DEBUG [RS:2;19945b63c2b4:37467 {}] regionserver.HRegionServer(2683): Master is not running yet 2024-11-26T14:31:10,739 WARN [RS:1;19945b63c2b4:40179 {}] regionserver.HRegionServer(841): reportForDuty failed; sleeping 100 ms and then retrying. 2024-11-26T14:31:10,739 WARN [RS:2;19945b63c2b4:37467 {}] regionserver.HRegionServer(841): reportForDuty failed; sleeping 100 ms and then retrying. 2024-11-26T14:31:10,739 WARN [RS:0;19945b63c2b4:35419 {}] regionserver.HRegionServer(841): reportForDuty failed; sleeping 100 ms and then retrying. 2024-11-26T14:31:10,740 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-26T14:31:10,740 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:31:10,741 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-26T14:31:10,741 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-26T14:31:10,744 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-26T14:31:10,744 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:31:10,745 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-26T14:31:10,746 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-26T14:31:10,748 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-26T14:31:10,748 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:31:10,749 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-26T14:31:10,750 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-26T14:31:10,751 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44999/user/jenkins/test-data/2de93217-0593-bca0-5c5f-879b6362c76f/data/hbase/meta/1588230740 2024-11-26T14:31:10,752 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44999/user/jenkins/test-data/2de93217-0593-bca0-5c5f-879b6362c76f/data/hbase/meta/1588230740 2024-11-26T14:31:10,755 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-26T14:31:10,755 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-26T14:31:10,756 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-26T14:31:10,758 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-26T14:31:10,762 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44999/user/jenkins/test-data/2de93217-0593-bca0-5c5f-879b6362c76f/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-26T14:31:10,764 INFO [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=69663444, jitterRate=0.038066208362579346}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-26T14:31:10,768 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740: Writing region info on filesystem at 1732631470731Initializing all the Stores at 1732631470733 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732631470733Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732631470733Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732631470733Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732631470733Cleaning up temporary data from old regions at 1732631470755 (+22 ms)Region opened successfully at 1732631470768 (+13 ms) 2024-11-26T14:31:10,768 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-26T14:31:10,768 INFO [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-26T14:31:10,768 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-26T14:31:10,769 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-26T14:31:10,769 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-26T14:31:10,770 INFO [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-26T14:31:10,770 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1732631470768Disabling compacts and flushes for region at 1732631470768Disabling writes for close at 1732631470769 (+1 ms)Writing region close event to WAL at 1732631470770 (+1 ms)Closed at 1732631470770 2024-11-26T14:31:10,773 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-26T14:31:10,774 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta 2024-11-26T14:31:10,779 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-11-26T14:31:10,787 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-26T14:31:10,790 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2024-11-26T14:31:10,840 INFO [RS:0;19945b63c2b4:35419 {}] regionserver.HRegionServer(2659): reportForDuty to master=19945b63c2b4,43577,1732631468672 with port=35419, startcode=1732631469384 2024-11-26T14:31:10,840 INFO [RS:2;19945b63c2b4:37467 {}] regionserver.HRegionServer(2659): reportForDuty to master=19945b63c2b4,43577,1732631468672 with port=37467, startcode=1732631469538 2024-11-26T14:31:10,840 INFO [RS:1;19945b63c2b4:40179 {}] regionserver.HRegionServer(2659): reportForDuty to master=19945b63c2b4,43577,1732631468672 with port=40179, startcode=1732631469496 2024-11-26T14:31:10,842 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=43577 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 19945b63c2b4,35419,1732631469384 2024-11-26T14:31:10,844 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=43577 {}] master.ServerManager(517): Registering regionserver=19945b63c2b4,35419,1732631469384 2024-11-26T14:31:10,852 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=43577 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 19945b63c2b4,37467,1732631469538 2024-11-26T14:31:10,852 DEBUG [RS:0;19945b63c2b4:35419 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:44999/user/jenkins/test-data/2de93217-0593-bca0-5c5f-879b6362c76f 2024-11-26T14:31:10,852 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=43577 {}] master.ServerManager(517): Registering regionserver=19945b63c2b4,37467,1732631469538 2024-11-26T14:31:10,853 DEBUG [RS:0;19945b63c2b4:35419 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:44999 2024-11-26T14:31:10,853 DEBUG [RS:0;19945b63c2b4:35419 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-26T14:31:10,855 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=43577 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 19945b63c2b4,40179,1732631469496 2024-11-26T14:31:10,856 DEBUG [RS:2;19945b63c2b4:37467 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:44999/user/jenkins/test-data/2de93217-0593-bca0-5c5f-879b6362c76f 2024-11-26T14:31:10,856 DEBUG [RS:2;19945b63c2b4:37467 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:44999 2024-11-26T14:31:10,856 DEBUG [RS:2;19945b63c2b4:37467 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-26T14:31:10,856 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=43577 {}] master.ServerManager(517): Registering regionserver=19945b63c2b4,40179,1732631469496 2024-11-26T14:31:10,859 DEBUG [RS:1;19945b63c2b4:40179 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:44999/user/jenkins/test-data/2de93217-0593-bca0-5c5f-879b6362c76f 2024-11-26T14:31:10,859 DEBUG [RS:1;19945b63c2b4:40179 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:44999 2024-11-26T14:31:10,859 DEBUG [RS:1;19945b63c2b4:40179 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-26T14:31:10,868 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43577-0x101778e13450000, quorum=127.0.0.1:49403, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-26T14:31:10,914 DEBUG [RS:0;19945b63c2b4:35419 {}] zookeeper.ZKUtil(111): regionserver:35419-0x101778e13450001, quorum=127.0.0.1:49403, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/19945b63c2b4,35419,1732631469384 2024-11-26T14:31:10,915 WARN [RS:0;19945b63c2b4:35419 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-26T14:31:10,915 DEBUG [RS:2;19945b63c2b4:37467 {}] zookeeper.ZKUtil(111): regionserver:37467-0x101778e13450003, quorum=127.0.0.1:49403, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/19945b63c2b4,37467,1732631469538 2024-11-26T14:31:10,915 INFO [RS:0;19945b63c2b4:35419 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-26T14:31:10,915 DEBUG [RS:1;19945b63c2b4:40179 {}] zookeeper.ZKUtil(111): regionserver:40179-0x101778e13450002, quorum=127.0.0.1:49403, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/19945b63c2b4,40179,1732631469496 2024-11-26T14:31:10,915 WARN [RS:1;19945b63c2b4:40179 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-26T14:31:10,915 WARN [RS:2;19945b63c2b4:37467 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-26T14:31:10,915 DEBUG [RS:0;19945b63c2b4:35419 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:44999/user/jenkins/test-data/2de93217-0593-bca0-5c5f-879b6362c76f/WALs/19945b63c2b4,35419,1732631469384 2024-11-26T14:31:10,915 INFO [RS:1;19945b63c2b4:40179 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-26T14:31:10,915 INFO [RS:2;19945b63c2b4:37467 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-26T14:31:10,915 DEBUG [RS:1;19945b63c2b4:40179 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:44999/user/jenkins/test-data/2de93217-0593-bca0-5c5f-879b6362c76f/WALs/19945b63c2b4,40179,1732631469496 2024-11-26T14:31:10,915 DEBUG [RS:2;19945b63c2b4:37467 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:44999/user/jenkins/test-data/2de93217-0593-bca0-5c5f-879b6362c76f/WALs/19945b63c2b4,37467,1732631469538 2024-11-26T14:31:10,916 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [19945b63c2b4,35419,1732631469384] 2024-11-26T14:31:10,917 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [19945b63c2b4,40179,1732631469496] 2024-11-26T14:31:10,917 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [19945b63c2b4,37467,1732631469538] 2024-11-26T14:31:10,940 INFO [RS:2;19945b63c2b4:37467 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-26T14:31:10,940 INFO [RS:0;19945b63c2b4:35419 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-26T14:31:10,940 INFO [RS:1;19945b63c2b4:40179 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-26T14:31:10,941 WARN [19945b63c2b4:43577 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-26T14:31:10,956 INFO [RS:2;19945b63c2b4:37467 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-26T14:31:10,956 INFO [RS:1;19945b63c2b4:40179 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-26T14:31:10,957 INFO [RS:0;19945b63c2b4:35419 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-26T14:31:10,961 INFO [RS:0;19945b63c2b4:35419 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-26T14:31:10,961 INFO [RS:1;19945b63c2b4:40179 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-26T14:31:10,962 INFO [RS:2;19945b63c2b4:37467 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-26T14:31:10,962 INFO [RS:1;19945b63c2b4:40179 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-26T14:31:10,962 INFO [RS:0;19945b63c2b4:35419 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-26T14:31:10,962 INFO [RS:2;19945b63c2b4:37467 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-26T14:31:10,963 INFO [RS:1;19945b63c2b4:40179 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-26T14:31:10,963 INFO [RS:2;19945b63c2b4:37467 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-26T14:31:10,963 INFO [RS:0;19945b63c2b4:35419 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-26T14:31:10,968 INFO [RS:0;19945b63c2b4:35419 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-26T14:31:10,968 INFO [RS:2;19945b63c2b4:37467 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-26T14:31:10,968 INFO [RS:1;19945b63c2b4:40179 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-26T14:31:10,970 INFO [RS:1;19945b63c2b4:40179 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-26T14:31:10,970 INFO [RS:2;19945b63c2b4:37467 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-26T14:31:10,970 INFO [RS:0;19945b63c2b4:35419 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-26T14:31:10,970 DEBUG [RS:0;19945b63c2b4:35419 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/19945b63c2b4:0, corePoolSize=1, maxPoolSize=1 2024-11-26T14:31:10,970 DEBUG [RS:2;19945b63c2b4:37467 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/19945b63c2b4:0, corePoolSize=1, maxPoolSize=1 2024-11-26T14:31:10,970 DEBUG [RS:1;19945b63c2b4:40179 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/19945b63c2b4:0, corePoolSize=1, maxPoolSize=1 2024-11-26T14:31:10,970 DEBUG [RS:0;19945b63c2b4:35419 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/19945b63c2b4:0, corePoolSize=1, maxPoolSize=1 2024-11-26T14:31:10,970 DEBUG [RS:1;19945b63c2b4:40179 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/19945b63c2b4:0, corePoolSize=1, maxPoolSize=1 2024-11-26T14:31:10,970 DEBUG [RS:2;19945b63c2b4:37467 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/19945b63c2b4:0, corePoolSize=1, maxPoolSize=1 2024-11-26T14:31:10,970 DEBUG [RS:0;19945b63c2b4:35419 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/19945b63c2b4:0, corePoolSize=1, maxPoolSize=1 2024-11-26T14:31:10,970 DEBUG [RS:1;19945b63c2b4:40179 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/19945b63c2b4:0, corePoolSize=1, maxPoolSize=1 2024-11-26T14:31:10,970 DEBUG [RS:0;19945b63c2b4:35419 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/19945b63c2b4:0, corePoolSize=1, maxPoolSize=1 2024-11-26T14:31:10,970 DEBUG [RS:2;19945b63c2b4:37467 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/19945b63c2b4:0, corePoolSize=1, maxPoolSize=1 2024-11-26T14:31:10,970 DEBUG [RS:1;19945b63c2b4:40179 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/19945b63c2b4:0, corePoolSize=1, maxPoolSize=1 2024-11-26T14:31:10,971 DEBUG [RS:0;19945b63c2b4:35419 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/19945b63c2b4:0, corePoolSize=1, maxPoolSize=1 2024-11-26T14:31:10,971 DEBUG [RS:2;19945b63c2b4:37467 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/19945b63c2b4:0, corePoolSize=1, maxPoolSize=1 2024-11-26T14:31:10,971 DEBUG [RS:1;19945b63c2b4:40179 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/19945b63c2b4:0, corePoolSize=1, maxPoolSize=1 2024-11-26T14:31:10,971 DEBUG [RS:0;19945b63c2b4:35419 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/19945b63c2b4:0, corePoolSize=2, maxPoolSize=2 2024-11-26T14:31:10,971 DEBUG [RS:1;19945b63c2b4:40179 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/19945b63c2b4:0, corePoolSize=2, maxPoolSize=2 2024-11-26T14:31:10,971 DEBUG [RS:2;19945b63c2b4:37467 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/19945b63c2b4:0, corePoolSize=1, maxPoolSize=1 2024-11-26T14:31:10,971 DEBUG [RS:0;19945b63c2b4:35419 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/19945b63c2b4:0, corePoolSize=1, maxPoolSize=1 2024-11-26T14:31:10,971 DEBUG [RS:1;19945b63c2b4:40179 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/19945b63c2b4:0, corePoolSize=1, maxPoolSize=1 2024-11-26T14:31:10,971 DEBUG [RS:2;19945b63c2b4:37467 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/19945b63c2b4:0, corePoolSize=2, maxPoolSize=2 2024-11-26T14:31:10,971 DEBUG [RS:0;19945b63c2b4:35419 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/19945b63c2b4:0, corePoolSize=1, maxPoolSize=1 2024-11-26T14:31:10,971 DEBUG [RS:1;19945b63c2b4:40179 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/19945b63c2b4:0, corePoolSize=1, maxPoolSize=1 2024-11-26T14:31:10,971 DEBUG [RS:2;19945b63c2b4:37467 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/19945b63c2b4:0, corePoolSize=1, maxPoolSize=1 2024-11-26T14:31:10,971 DEBUG [RS:1;19945b63c2b4:40179 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/19945b63c2b4:0, corePoolSize=1, maxPoolSize=1 2024-11-26T14:31:10,971 DEBUG [RS:0;19945b63c2b4:35419 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/19945b63c2b4:0, corePoolSize=1, maxPoolSize=1 2024-11-26T14:31:10,971 DEBUG [RS:1;19945b63c2b4:40179 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/19945b63c2b4:0, corePoolSize=1, maxPoolSize=1 2024-11-26T14:31:10,971 DEBUG [RS:0;19945b63c2b4:35419 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/19945b63c2b4:0, corePoolSize=1, maxPoolSize=1 2024-11-26T14:31:10,971 DEBUG [RS:2;19945b63c2b4:37467 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/19945b63c2b4:0, corePoolSize=1, maxPoolSize=1 2024-11-26T14:31:10,971 DEBUG [RS:1;19945b63c2b4:40179 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/19945b63c2b4:0, corePoolSize=1, maxPoolSize=1 2024-11-26T14:31:10,971 DEBUG [RS:0;19945b63c2b4:35419 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/19945b63c2b4:0, corePoolSize=1, maxPoolSize=1 2024-11-26T14:31:10,971 DEBUG [RS:2;19945b63c2b4:37467 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/19945b63c2b4:0, corePoolSize=1, maxPoolSize=1 2024-11-26T14:31:10,971 DEBUG [RS:1;19945b63c2b4:40179 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/19945b63c2b4:0, corePoolSize=1, maxPoolSize=1 2024-11-26T14:31:10,971 DEBUG [RS:0;19945b63c2b4:35419 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/19945b63c2b4:0, corePoolSize=1, maxPoolSize=1 2024-11-26T14:31:10,972 DEBUG [RS:2;19945b63c2b4:37467 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/19945b63c2b4:0, corePoolSize=1, maxPoolSize=1 2024-11-26T14:31:10,972 DEBUG [RS:1;19945b63c2b4:40179 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/19945b63c2b4:0, corePoolSize=3, maxPoolSize=3 2024-11-26T14:31:10,972 DEBUG [RS:0;19945b63c2b4:35419 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/19945b63c2b4:0, corePoolSize=3, maxPoolSize=3 2024-11-26T14:31:10,972 DEBUG [RS:1;19945b63c2b4:40179 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0, corePoolSize=3, maxPoolSize=3 2024-11-26T14:31:10,972 DEBUG [RS:0;19945b63c2b4:35419 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0, corePoolSize=3, maxPoolSize=3 2024-11-26T14:31:10,972 DEBUG [RS:2;19945b63c2b4:37467 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/19945b63c2b4:0, corePoolSize=1, maxPoolSize=1 2024-11-26T14:31:10,972 DEBUG [RS:2;19945b63c2b4:37467 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/19945b63c2b4:0, corePoolSize=1, maxPoolSize=1 2024-11-26T14:31:10,972 DEBUG [RS:2;19945b63c2b4:37467 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/19945b63c2b4:0, corePoolSize=3, maxPoolSize=3 2024-11-26T14:31:10,972 DEBUG [RS:2;19945b63c2b4:37467 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/19945b63c2b4:0, corePoolSize=3, maxPoolSize=3 2024-11-26T14:31:10,975 INFO [RS:1;19945b63c2b4:40179 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-26T14:31:10,975 INFO [RS:1;19945b63c2b4:40179 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-26T14:31:10,975 INFO [RS:1;19945b63c2b4:40179 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-26T14:31:10,975 INFO [RS:1;19945b63c2b4:40179 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-26T14:31:10,975 INFO [RS:1;19945b63c2b4:40179 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-26T14:31:10,976 INFO [RS:1;19945b63c2b4:40179 {}] hbase.ChoreService(168): Chore ScheduledChore name=19945b63c2b4,40179,1732631469496-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-26T14:31:10,976 INFO [RS:0;19945b63c2b4:35419 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-26T14:31:10,976 INFO [RS:2;19945b63c2b4:37467 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-26T14:31:10,976 INFO [RS:0;19945b63c2b4:35419 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-26T14:31:10,976 INFO [RS:2;19945b63c2b4:37467 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-26T14:31:10,976 INFO [RS:0;19945b63c2b4:35419 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-26T14:31:10,976 INFO [RS:2;19945b63c2b4:37467 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-26T14:31:10,976 INFO [RS:0;19945b63c2b4:35419 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-26T14:31:10,976 INFO [RS:2;19945b63c2b4:37467 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-26T14:31:10,976 INFO [RS:0;19945b63c2b4:35419 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-26T14:31:10,976 INFO [RS:2;19945b63c2b4:37467 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-26T14:31:10,976 INFO [RS:0;19945b63c2b4:35419 {}] hbase.ChoreService(168): Chore ScheduledChore name=19945b63c2b4,35419,1732631469384-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-26T14:31:10,976 INFO [RS:2;19945b63c2b4:37467 {}] hbase.ChoreService(168): Chore ScheduledChore name=19945b63c2b4,37467,1732631469538-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-26T14:31:10,995 INFO [RS:1;19945b63c2b4:40179 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-26T14:31:10,995 INFO [RS:2;19945b63c2b4:37467 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-26T14:31:10,997 INFO [RS:1;19945b63c2b4:40179 {}] hbase.ChoreService(168): Chore ScheduledChore name=19945b63c2b4,40179,1732631469496-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-26T14:31:10,997 INFO [RS:2;19945b63c2b4:37467 {}] hbase.ChoreService(168): Chore ScheduledChore name=19945b63c2b4,37467,1732631469538-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-26T14:31:10,998 INFO [RS:2;19945b63c2b4:37467 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-26T14:31:10,998 INFO [RS:1;19945b63c2b4:40179 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-26T14:31:10,998 INFO [RS:1;19945b63c2b4:40179 {}] regionserver.Replication(171): 19945b63c2b4,40179,1732631469496 started 2024-11-26T14:31:10,998 INFO [RS:2;19945b63c2b4:37467 {}] regionserver.Replication(171): 19945b63c2b4,37467,1732631469538 started 2024-11-26T14:31:10,999 INFO [RS:0;19945b63c2b4:35419 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-26T14:31:11,000 INFO [RS:0;19945b63c2b4:35419 {}] hbase.ChoreService(168): Chore ScheduledChore name=19945b63c2b4,35419,1732631469384-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-26T14:31:11,000 INFO [RS:0;19945b63c2b4:35419 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-26T14:31:11,000 INFO [RS:0;19945b63c2b4:35419 {}] regionserver.Replication(171): 19945b63c2b4,35419,1732631469384 started 2024-11-26T14:31:11,016 INFO [RS:2;19945b63c2b4:37467 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-26T14:31:11,016 INFO [RS:1;19945b63c2b4:40179 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-26T14:31:11,016 INFO [RS:2;19945b63c2b4:37467 {}] regionserver.HRegionServer(1482): Serving as 19945b63c2b4,37467,1732631469538, RpcServer on 19945b63c2b4/172.17.0.3:37467, sessionid=0x101778e13450003 2024-11-26T14:31:11,016 INFO [RS:1;19945b63c2b4:40179 {}] regionserver.HRegionServer(1482): Serving as 19945b63c2b4,40179,1732631469496, RpcServer on 19945b63c2b4/172.17.0.3:40179, sessionid=0x101778e13450002 2024-11-26T14:31:11,017 DEBUG [RS:2;19945b63c2b4:37467 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-26T14:31:11,017 DEBUG [RS:1;19945b63c2b4:40179 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-26T14:31:11,017 DEBUG [RS:2;19945b63c2b4:37467 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 19945b63c2b4,37467,1732631469538 2024-11-26T14:31:11,017 DEBUG [RS:1;19945b63c2b4:40179 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 19945b63c2b4,40179,1732631469496 2024-11-26T14:31:11,017 DEBUG [RS:2;19945b63c2b4:37467 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '19945b63c2b4,37467,1732631469538' 2024-11-26T14:31:11,017 DEBUG [RS:1;19945b63c2b4:40179 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '19945b63c2b4,40179,1732631469496' 2024-11-26T14:31:11,017 DEBUG [RS:1;19945b63c2b4:40179 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-26T14:31:11,017 DEBUG [RS:2;19945b63c2b4:37467 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-26T14:31:11,018 DEBUG [RS:1;19945b63c2b4:40179 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-26T14:31:11,018 DEBUG [RS:2;19945b63c2b4:37467 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-26T14:31:11,018 DEBUG [RS:1;19945b63c2b4:40179 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-26T14:31:11,018 DEBUG [RS:2;19945b63c2b4:37467 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-26T14:31:11,019 DEBUG [RS:1;19945b63c2b4:40179 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-26T14:31:11,019 DEBUG [RS:2;19945b63c2b4:37467 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-26T14:31:11,019 DEBUG [RS:1;19945b63c2b4:40179 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 19945b63c2b4,40179,1732631469496 2024-11-26T14:31:11,019 DEBUG [RS:2;19945b63c2b4:37467 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 19945b63c2b4,37467,1732631469538 2024-11-26T14:31:11,019 DEBUG [RS:2;19945b63c2b4:37467 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '19945b63c2b4,37467,1732631469538' 2024-11-26T14:31:11,019 DEBUG [RS:1;19945b63c2b4:40179 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '19945b63c2b4,40179,1732631469496' 2024-11-26T14:31:11,019 DEBUG [RS:2;19945b63c2b4:37467 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-26T14:31:11,019 DEBUG [RS:1;19945b63c2b4:40179 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-26T14:31:11,019 DEBUG [RS:1;19945b63c2b4:40179 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-26T14:31:11,020 DEBUG [RS:2;19945b63c2b4:37467 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-26T14:31:11,020 DEBUG [RS:1;19945b63c2b4:40179 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-26T14:31:11,020 DEBUG [RS:2;19945b63c2b4:37467 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-26T14:31:11,020 INFO [RS:1;19945b63c2b4:40179 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-26T14:31:11,020 INFO [RS:2;19945b63c2b4:37467 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-26T14:31:11,020 INFO [RS:1;19945b63c2b4:40179 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-26T14:31:11,020 INFO [RS:2;19945b63c2b4:37467 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-26T14:31:11,022 INFO [RS:0;19945b63c2b4:35419 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-26T14:31:11,023 INFO [RS:0;19945b63c2b4:35419 {}] regionserver.HRegionServer(1482): Serving as 19945b63c2b4,35419,1732631469384, RpcServer on 19945b63c2b4/172.17.0.3:35419, sessionid=0x101778e13450001 2024-11-26T14:31:11,023 DEBUG [RS:0;19945b63c2b4:35419 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-26T14:31:11,023 DEBUG [RS:0;19945b63c2b4:35419 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 19945b63c2b4,35419,1732631469384 2024-11-26T14:31:11,023 DEBUG [RS:0;19945b63c2b4:35419 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '19945b63c2b4,35419,1732631469384' 2024-11-26T14:31:11,023 DEBUG [RS:0;19945b63c2b4:35419 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-26T14:31:11,024 DEBUG [RS:0;19945b63c2b4:35419 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-26T14:31:11,024 DEBUG [RS:0;19945b63c2b4:35419 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-26T14:31:11,025 DEBUG [RS:0;19945b63c2b4:35419 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-26T14:31:11,025 DEBUG [RS:0;19945b63c2b4:35419 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 19945b63c2b4,35419,1732631469384 2024-11-26T14:31:11,025 DEBUG [RS:0;19945b63c2b4:35419 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '19945b63c2b4,35419,1732631469384' 2024-11-26T14:31:11,025 DEBUG [RS:0;19945b63c2b4:35419 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-26T14:31:11,025 DEBUG [RS:0;19945b63c2b4:35419 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-26T14:31:11,026 DEBUG [RS:0;19945b63c2b4:35419 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-26T14:31:11,026 INFO [RS:0;19945b63c2b4:35419 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-26T14:31:11,026 INFO [RS:0;19945b63c2b4:35419 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-26T14:31:11,125 INFO [RS:1;19945b63c2b4:40179 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-26T14:31:11,125 INFO [RS:2;19945b63c2b4:37467 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-26T14:31:11,127 INFO [RS:0;19945b63c2b4:35419 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-26T14:31:11,128 INFO [RS:2;19945b63c2b4:37467 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=19945b63c2b4%2C37467%2C1732631469538, suffix=, logDir=hdfs://localhost:44999/user/jenkins/test-data/2de93217-0593-bca0-5c5f-879b6362c76f/WALs/19945b63c2b4,37467,1732631469538, archiveDir=hdfs://localhost:44999/user/jenkins/test-data/2de93217-0593-bca0-5c5f-879b6362c76f/oldWALs, maxLogs=32 2024-11-26T14:31:11,128 INFO [RS:1;19945b63c2b4:40179 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=19945b63c2b4%2C40179%2C1732631469496, suffix=, logDir=hdfs://localhost:44999/user/jenkins/test-data/2de93217-0593-bca0-5c5f-879b6362c76f/WALs/19945b63c2b4,40179,1732631469496, archiveDir=hdfs://localhost:44999/user/jenkins/test-data/2de93217-0593-bca0-5c5f-879b6362c76f/oldWALs, maxLogs=32 2024-11-26T14:31:11,129 INFO [RS:0;19945b63c2b4:35419 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=19945b63c2b4%2C35419%2C1732631469384, suffix=, logDir=hdfs://localhost:44999/user/jenkins/test-data/2de93217-0593-bca0-5c5f-879b6362c76f/WALs/19945b63c2b4,35419,1732631469384, archiveDir=hdfs://localhost:44999/user/jenkins/test-data/2de93217-0593-bca0-5c5f-879b6362c76f/oldWALs, maxLogs=32 2024-11-26T14:31:11,145 DEBUG [RS:1;19945b63c2b4:40179 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/2de93217-0593-bca0-5c5f-879b6362c76f/WALs/19945b63c2b4,40179,1732631469496/19945b63c2b4%2C40179%2C1732631469496.1732631471133, exclude list is [], retry=0 2024-11-26T14:31:11,145 DEBUG [RS:2;19945b63c2b4:37467 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/2de93217-0593-bca0-5c5f-879b6362c76f/WALs/19945b63c2b4,37467,1732631469538/19945b63c2b4%2C37467%2C1732631469538.1732631471132, exclude list is [], retry=0 2024-11-26T14:31:11,145 DEBUG [RS:0;19945b63c2b4:35419 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/2de93217-0593-bca0-5c5f-879b6362c76f/WALs/19945b63c2b4,35419,1732631469384/19945b63c2b4%2C35419%2C1732631469384.1732631471132, exclude list is [], retry=0 2024-11-26T14:31:11,150 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36277,DS-9cb9383d-a6c8-4ba6-83b3-6706331a6539,DISK] 2024-11-26T14:31:11,151 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38147,DS-44b12d44-38d0-4c76-b56f-3ca8d8caa398,DISK] 2024-11-26T14:31:11,151 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36277,DS-9cb9383d-a6c8-4ba6-83b3-6706331a6539,DISK] 2024-11-26T14:31:11,151 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36307,DS-dd37de78-b3d5-4d7c-8223-4b15101c21bf,DISK] 2024-11-26T14:31:11,151 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36307,DS-dd37de78-b3d5-4d7c-8223-4b15101c21bf,DISK] 2024-11-26T14:31:11,152 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38147,DS-44b12d44-38d0-4c76-b56f-3ca8d8caa398,DISK] 2024-11-26T14:31:11,152 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36307,DS-dd37de78-b3d5-4d7c-8223-4b15101c21bf,DISK] 2024-11-26T14:31:11,153 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38147,DS-44b12d44-38d0-4c76-b56f-3ca8d8caa398,DISK] 2024-11-26T14:31:11,153 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36277,DS-9cb9383d-a6c8-4ba6-83b3-6706331a6539,DISK] 2024-11-26T14:31:11,161 INFO [RS:1;19945b63c2b4:40179 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/2de93217-0593-bca0-5c5f-879b6362c76f/WALs/19945b63c2b4,40179,1732631469496/19945b63c2b4%2C40179%2C1732631469496.1732631471133 2024-11-26T14:31:11,166 INFO [RS:2;19945b63c2b4:37467 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/2de93217-0593-bca0-5c5f-879b6362c76f/WALs/19945b63c2b4,37467,1732631469538/19945b63c2b4%2C37467%2C1732631469538.1732631471132 2024-11-26T14:31:11,167 DEBUG [RS:1;19945b63c2b4:40179 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:40449:40449),(127.0.0.1/127.0.0.1:35081:35081),(127.0.0.1/127.0.0.1:34431:34431)] 2024-11-26T14:31:11,169 DEBUG [RS:2;19945b63c2b4:37467 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:40449:40449),(127.0.0.1/127.0.0.1:34431:34431),(127.0.0.1/127.0.0.1:35081:35081)] 2024-11-26T14:31:11,171 INFO [RS:0;19945b63c2b4:35419 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/2de93217-0593-bca0-5c5f-879b6362c76f/WALs/19945b63c2b4,35419,1732631469384/19945b63c2b4%2C35419%2C1732631469384.1732631471132 2024-11-26T14:31:11,171 DEBUG [RS:0;19945b63c2b4:35419 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:34431:34431),(127.0.0.1/127.0.0.1:40449:40449),(127.0.0.1/127.0.0.1:35081:35081)] 2024-11-26T14:31:11,194 DEBUG [19945b63c2b4:43577 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=3, allServersCount=3 2024-11-26T14:31:11,201 DEBUG [19945b63c2b4:43577 {}] balancer.BalancerClusterState(204): Hosts are {19945b63c2b4=0} racks are {/default-rack=0} 2024-11-26T14:31:11,207 DEBUG [19945b63c2b4:43577 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-11-26T14:31:11,207 DEBUG [19945b63c2b4:43577 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-11-26T14:31:11,207 DEBUG [19945b63c2b4:43577 {}] balancer.BalancerClusterState(303): server 2 has 0 regions 2024-11-26T14:31:11,207 DEBUG [19945b63c2b4:43577 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-11-26T14:31:11,207 DEBUG [19945b63c2b4:43577 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-11-26T14:31:11,207 DEBUG [19945b63c2b4:43577 {}] balancer.BalancerClusterState(310): server 2 is on host 0 2024-11-26T14:31:11,207 INFO [19945b63c2b4:43577 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-11-26T14:31:11,207 INFO [19945b63c2b4:43577 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-11-26T14:31:11,208 INFO [19945b63c2b4:43577 {}] balancer.BalancerClusterState(321): server 2 is on rack 0 2024-11-26T14:31:11,208 DEBUG [19945b63c2b4:43577 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-11-26T14:31:11,214 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=19945b63c2b4,37467,1732631469538 2024-11-26T14:31:11,219 INFO [PEWorker-2 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 19945b63c2b4,37467,1732631469538, state=OPENING 2024-11-26T14:31:11,240 DEBUG [PEWorker-2 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it 2024-11-26T14:31:11,251 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35419-0x101778e13450001, quorum=127.0.0.1:49403, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-26T14:31:11,251 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43577-0x101778e13450000, quorum=127.0.0.1:49403, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-26T14:31:11,251 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40179-0x101778e13450002, quorum=127.0.0.1:49403, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-26T14:31:11,251 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37467-0x101778e13450003, quorum=127.0.0.1:49403, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-26T14:31:11,253 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-26T14:31:11,253 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-26T14:31:11,253 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-26T14:31:11,254 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-26T14:31:11,256 DEBUG [PEWorker-2 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-26T14:31:11,259 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=19945b63c2b4,37467,1732631469538}] 2024-11-26T14:31:11,432 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-26T14:31:11,434 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:40833, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-26T14:31:11,449 INFO [RS_OPEN_META-regionserver/19945b63c2b4:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-11-26T14:31:11,450 INFO [RS_OPEN_META-regionserver/19945b63c2b4:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-26T14:31:11,450 INFO [RS_OPEN_META-regionserver/19945b63c2b4:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-11-26T14:31:11,454 INFO [RS_OPEN_META-regionserver/19945b63c2b4:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=19945b63c2b4%2C37467%2C1732631469538.meta, suffix=.meta, logDir=hdfs://localhost:44999/user/jenkins/test-data/2de93217-0593-bca0-5c5f-879b6362c76f/WALs/19945b63c2b4,37467,1732631469538, archiveDir=hdfs://localhost:44999/user/jenkins/test-data/2de93217-0593-bca0-5c5f-879b6362c76f/oldWALs, maxLogs=32 2024-11-26T14:31:11,471 DEBUG [RS_OPEN_META-regionserver/19945b63c2b4:0-0 {event_type=M_RS_OPEN_META, pid=3}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/2de93217-0593-bca0-5c5f-879b6362c76f/WALs/19945b63c2b4,37467,1732631469538/19945b63c2b4%2C37467%2C1732631469538.meta.1732631471457.meta, exclude list is [], retry=0 2024-11-26T14:31:11,475 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38147,DS-44b12d44-38d0-4c76-b56f-3ca8d8caa398,DISK] 2024-11-26T14:31:11,475 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36307,DS-dd37de78-b3d5-4d7c-8223-4b15101c21bf,DISK] 2024-11-26T14:31:11,475 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36277,DS-9cb9383d-a6c8-4ba6-83b3-6706331a6539,DISK] 2024-11-26T14:31:11,478 INFO [RS_OPEN_META-regionserver/19945b63c2b4:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/2de93217-0593-bca0-5c5f-879b6362c76f/WALs/19945b63c2b4,37467,1732631469538/19945b63c2b4%2C37467%2C1732631469538.meta.1732631471457.meta 2024-11-26T14:31:11,478 DEBUG [RS_OPEN_META-regionserver/19945b63c2b4:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:35081:35081),(127.0.0.1/127.0.0.1:34431:34431),(127.0.0.1/127.0.0.1:40449:40449)] 2024-11-26T14:31:11,479 DEBUG [RS_OPEN_META-regionserver/19945b63c2b4:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-11-26T14:31:11,480 DEBUG [RS_OPEN_META-regionserver/19945b63c2b4:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-11-26T14:31:11,483 DEBUG [RS_OPEN_META-regionserver/19945b63c2b4:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-11-26T14:31:11,487 INFO [RS_OPEN_META-regionserver/19945b63c2b4:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-11-26T14:31:11,491 DEBUG [RS_OPEN_META-regionserver/19945b63c2b4:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-11-26T14:31:11,492 DEBUG [RS_OPEN_META-regionserver/19945b63c2b4:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-26T14:31:11,492 DEBUG [RS_OPEN_META-regionserver/19945b63c2b4:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-11-26T14:31:11,492 DEBUG [RS_OPEN_META-regionserver/19945b63c2b4:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-11-26T14:31:11,495 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-26T14:31:11,497 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-26T14:31:11,497 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:31:11,498 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-26T14:31:11,498 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-26T14:31:11,500 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-26T14:31:11,500 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:31:11,501 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-26T14:31:11,501 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-26T14:31:11,502 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-26T14:31:11,502 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:31:11,503 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-26T14:31:11,503 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-26T14:31:11,505 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-26T14:31:11,505 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:31:11,505 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-26T14:31:11,506 DEBUG [RS_OPEN_META-regionserver/19945b63c2b4:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-26T14:31:11,507 DEBUG [RS_OPEN_META-regionserver/19945b63c2b4:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44999/user/jenkins/test-data/2de93217-0593-bca0-5c5f-879b6362c76f/data/hbase/meta/1588230740 2024-11-26T14:31:11,510 DEBUG [RS_OPEN_META-regionserver/19945b63c2b4:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44999/user/jenkins/test-data/2de93217-0593-bca0-5c5f-879b6362c76f/data/hbase/meta/1588230740 2024-11-26T14:31:11,512 DEBUG [RS_OPEN_META-regionserver/19945b63c2b4:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-26T14:31:11,512 DEBUG [RS_OPEN_META-regionserver/19945b63c2b4:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-26T14:31:11,513 DEBUG [RS_OPEN_META-regionserver/19945b63c2b4:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-26T14:31:11,516 DEBUG [RS_OPEN_META-regionserver/19945b63c2b4:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-26T14:31:11,517 INFO [RS_OPEN_META-regionserver/19945b63c2b4:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=66336100, jitterRate=-0.01151508092880249}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-26T14:31:11,517 DEBUG [RS_OPEN_META-regionserver/19945b63c2b4:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-11-26T14:31:11,518 DEBUG [RS_OPEN_META-regionserver/19945b63c2b4:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1732631471492Writing region info on filesystem at 1732631471493 (+1 ms)Initializing all the Stores at 1732631471495 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732631471495Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732631471495Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732631471495Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732631471495Cleaning up temporary data from old regions at 1732631471512 (+17 ms)Running coprocessor post-open hooks at 1732631471517 (+5 ms)Region opened successfully at 1732631471518 (+1 ms) 2024-11-26T14:31:11,525 INFO [RS_OPEN_META-regionserver/19945b63c2b4:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1732631471425 2024-11-26T14:31:11,536 DEBUG [RS_OPEN_META-regionserver/19945b63c2b4:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-11-26T14:31:11,536 INFO [RS_OPEN_META-regionserver/19945b63c2b4:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-11-26T14:31:11,538 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=19945b63c2b4,37467,1732631469538 2024-11-26T14:31:11,540 INFO [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 19945b63c2b4,37467,1732631469538, state=OPEN 2024-11-26T14:31:11,552 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43577-0x101778e13450000, quorum=127.0.0.1:49403, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-26T14:31:11,552 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37467-0x101778e13450003, quorum=127.0.0.1:49403, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-26T14:31:11,552 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35419-0x101778e13450001, quorum=127.0.0.1:49403, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-26T14:31:11,552 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-26T14:31:11,553 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-26T14:31:11,552 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40179-0x101778e13450002, quorum=127.0.0.1:49403, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-26T14:31:11,552 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-26T14:31:11,553 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-26T14:31:11,553 DEBUG [PEWorker-5 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=19945b63c2b4,37467,1732631469538 2024-11-26T14:31:11,560 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2 2024-11-26T14:31:11,561 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=19945b63c2b4,37467,1732631469538 in 294 msec 2024-11-26T14:31:11,569 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1 2024-11-26T14:31:11,569 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 784 msec 2024-11-26T14:31:11,570 DEBUG [PEWorker-3 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-26T14:31:11,570 INFO [PEWorker-3 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces 2024-11-26T14:31:11,590 DEBUG [PEWorker-3 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-26T14:31:11,591 DEBUG [PEWorker-3 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=19945b63c2b4,37467,1732631469538, seqNum=-1] 2024-11-26T14:31:11,611 DEBUG [PEWorker-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-26T14:31:11,614 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:49959, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-26T14:31:11,635 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 1.0470 sec 2024-11-26T14:31:11,635 INFO [master/19945b63c2b4:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1732631471635, completionTime=-1 2024-11-26T14:31:11,639 INFO [master/19945b63c2b4:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=3; waited=0ms, expected min=3 server(s), max=3 server(s), master is running 2024-11-26T14:31:11,639 DEBUG [master/19945b63c2b4:0:becomeActiveMaster {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-26T14:31:11,686 INFO [master/19945b63c2b4:0:becomeActiveMaster {}] assignment.AssignmentManager(1776): Number of RegionServers=3 2024-11-26T14:31:11,686 INFO [master/19945b63c2b4:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732631531686 2024-11-26T14:31:11,686 INFO [master/19945b63c2b4:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732631591686 2024-11-26T14:31:11,686 INFO [master/19945b63c2b4:0:becomeActiveMaster {}] assignment.AssignmentManager(1783): Joined the cluster in 47 msec 2024-11-26T14:31:11,688 DEBUG [master/19945b63c2b4:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 1588230740 changed from -1.0 to 0.0, refreshing cache 2024-11-26T14:31:11,694 INFO [master/19945b63c2b4:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=19945b63c2b4,43577,1732631468672-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-26T14:31:11,695 INFO [master/19945b63c2b4:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=19945b63c2b4,43577,1732631468672-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-26T14:31:11,695 INFO [master/19945b63c2b4:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=19945b63c2b4,43577,1732631468672-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-26T14:31:11,696 INFO [master/19945b63c2b4:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-19945b63c2b4:43577, period=300000, unit=MILLISECONDS is enabled. 2024-11-26T14:31:11,697 INFO [master/19945b63c2b4:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2024-11-26T14:31:11,697 INFO [master/19945b63c2b4:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled. 2024-11-26T14:31:11,703 DEBUG [master/19945b63c2b4:0.Chore.1 {}] janitor.CatalogJanitor(180): 2024-11-26T14:31:11,735 INFO [master/19945b63c2b4:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 2.082sec 2024-11-26T14:31:11,737 INFO [master/19945b63c2b4:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2024-11-26T14:31:11,738 INFO [master/19945b63c2b4:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2024-11-26T14:31:11,738 INFO [master/19945b63c2b4:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2024-11-26T14:31:11,739 INFO [master/19945b63c2b4:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2024-11-26T14:31:11,739 INFO [master/19945b63c2b4:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2024-11-26T14:31:11,739 INFO [master/19945b63c2b4:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=19945b63c2b4,43577,1732631468672-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-26T14:31:11,740 INFO [master/19945b63c2b4:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=19945b63c2b4,43577,1732631468672-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2024-11-26T14:31:11,744 DEBUG [master/19945b63c2b4:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds 2024-11-26T14:31:11,745 INFO [master/19945b63c2b4:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2024-11-26T14:31:11,745 INFO [master/19945b63c2b4:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=19945b63c2b4,43577,1732631468672-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-26T14:31:11,782 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@78619a57, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T14:31:11,783 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 19945b63c2b4,43577,-1 for getting cluster id 2024-11-26T14:31:11,785 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-26T14:31:11,792 DEBUG [HMaster-EventLoopGroup-2-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '85e6cd0d-a40d-403d-9d6e-04753ee5ca5e' 2024-11-26T14:31:11,794 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-26T14:31:11,794 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "85e6cd0d-a40d-403d-9d6e-04753ee5ca5e" 2024-11-26T14:31:11,794 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@69727455, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T14:31:11,794 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [19945b63c2b4,43577,-1] 2024-11-26T14:31:11,796 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-26T14:31:11,798 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T14:31:11,799 INFO [HMaster-EventLoopGroup-2-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:45664, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-26T14:31:11,802 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1dd02f5b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T14:31:11,802 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-26T14:31:11,809 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=19945b63c2b4,37467,1732631469538, seqNum=-1] 2024-11-26T14:31:11,809 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-26T14:31:11,812 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:34842, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-26T14:31:11,830 INFO [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=19945b63c2b4,43577,1732631468672 2024-11-26T14:31:11,831 INFO [Time-limited test {}] wal.AbstractTestWALReplay(147): hbase.rootdir=hdfs://localhost:44999/hbase 2024-11-26T14:31:11,845 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestAsyncWALReplayCompressed#testNameConflictWhenSplit0 Thread=367, OpenFileDescriptor=611, MaxFileDescriptor=1048576, SystemLoadAverage=224, ProcessCount=11, AvailableMemoryMB=9469 2024-11-26T14:31:11,862 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-26T14:31:11,865 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-26T14:31:11,866 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-26T14:31:11,870 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-33454757, suffix=, logDir=hdfs://localhost:44999/hbase/WALs/hregion-33454757, archiveDir=hdfs://localhost:44999/hbase/oldWALs, maxLogs=32 2024-11-26T14:31:11,884 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-33454757/hregion-33454757.1732631471872, exclude list is [], retry=0 2024-11-26T14:31:11,888 DEBUG [AsyncFSWAL-8-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36277,DS-9cb9383d-a6c8-4ba6-83b3-6706331a6539,DISK] 2024-11-26T14:31:11,888 DEBUG [AsyncFSWAL-8-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36307,DS-dd37de78-b3d5-4d7c-8223-4b15101c21bf,DISK] 2024-11-26T14:31:11,889 DEBUG [AsyncFSWAL-8-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38147,DS-44b12d44-38d0-4c76-b56f-3ca8d8caa398,DISK] 2024-11-26T14:31:11,893 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-33454757/hregion-33454757.1732631471872 2024-11-26T14:31:11,893 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:40449:40449),(127.0.0.1/127.0.0.1:34431:34431),(127.0.0.1/127.0.0.1:35081:35081)] 2024-11-26T14:31:11,893 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => 0646c372d90fdcf989fc1feb0b0b253d, NAME => 'testReplayEditsWrittenIntoWAL,,1732631471863.0646c372d90fdcf989fc1feb0b0b253d.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testReplayEditsWrittenIntoWAL', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:44999/hbase 2024-11-26T14:31:11,906 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36277 is added to blk_1073741838_1014 (size=64) 2024-11-26T14:31:11,908 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36307 is added to blk_1073741838_1014 (size=64) 2024-11-26T14:31:11,908 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38147 is added to blk_1073741838_1014 (size=64) 2024-11-26T14:31:11,909 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenIntoWAL,,1732631471863.0646c372d90fdcf989fc1feb0b0b253d.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-26T14:31:11,911 INFO [StoreOpener-0646c372d90fdcf989fc1feb0b0b253d-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 0646c372d90fdcf989fc1feb0b0b253d 2024-11-26T14:31:11,914 INFO [StoreOpener-0646c372d90fdcf989fc1feb0b0b253d-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 0646c372d90fdcf989fc1feb0b0b253d columnFamilyName a 2024-11-26T14:31:11,914 DEBUG [StoreOpener-0646c372d90fdcf989fc1feb0b0b253d-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:31:11,915 INFO [StoreOpener-0646c372d90fdcf989fc1feb0b0b253d-1 {}] regionserver.HStore(327): Store=0646c372d90fdcf989fc1feb0b0b253d/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-26T14:31:11,915 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 0646c372d90fdcf989fc1feb0b0b253d 2024-11-26T14:31:11,917 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44999/hbase/data/default/testReplayEditsWrittenIntoWAL/0646c372d90fdcf989fc1feb0b0b253d 2024-11-26T14:31:11,917 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44999/hbase/data/default/testReplayEditsWrittenIntoWAL/0646c372d90fdcf989fc1feb0b0b253d 2024-11-26T14:31:11,918 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 0646c372d90fdcf989fc1feb0b0b253d 2024-11-26T14:31:11,918 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 0646c372d90fdcf989fc1feb0b0b253d 2024-11-26T14:31:11,921 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 0646c372d90fdcf989fc1feb0b0b253d 2024-11-26T14:31:11,925 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44999/hbase/data/default/testReplayEditsWrittenIntoWAL/0646c372d90fdcf989fc1feb0b0b253d/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-26T14:31:11,926 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 0646c372d90fdcf989fc1feb0b0b253d; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=69408562, jitterRate=0.03426817059516907}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-26T14:31:11,927 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 0646c372d90fdcf989fc1feb0b0b253d: Writing region info on filesystem at 1732631471909Initializing all the Stores at 1732631471910 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732631471911 (+1 ms)Cleaning up temporary data from old regions at 1732631471918 (+7 ms)Region opened successfully at 1732631471927 (+9 ms) 2024-11-26T14:31:11,927 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 0646c372d90fdcf989fc1feb0b0b253d, disabling compactions & flushes 2024-11-26T14:31:11,927 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsWrittenIntoWAL,,1732631471863.0646c372d90fdcf989fc1feb0b0b253d. 2024-11-26T14:31:11,927 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsWrittenIntoWAL,,1732631471863.0646c372d90fdcf989fc1feb0b0b253d. 2024-11-26T14:31:11,927 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsWrittenIntoWAL,,1732631471863.0646c372d90fdcf989fc1feb0b0b253d. after waiting 0 ms 2024-11-26T14:31:11,927 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsWrittenIntoWAL,,1732631471863.0646c372d90fdcf989fc1feb0b0b253d. 2024-11-26T14:31:11,928 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsWrittenIntoWAL,,1732631471863.0646c372d90fdcf989fc1feb0b0b253d. 2024-11-26T14:31:11,928 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 0646c372d90fdcf989fc1feb0b0b253d: Waiting for close lock at 1732631471927Disabling compacts and flushes for region at 1732631471927Disabling writes for close at 1732631471927Writing region close event to WAL at 1732631471928 (+1 ms)Closed at 1732631471928 2024-11-26T14:31:11,933 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36277 is added to blk_1073741837_1013 (size=93) 2024-11-26T14:31:11,933 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36307 is added to blk_1073741837_1013 (size=93) 2024-11-26T14:31:11,933 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38147 is added to blk_1073741837_1013 (size=93) 2024-11-26T14:31:11,939 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-11-26T14:31:11,940 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-33454757:(num 1732631471872) 2024-11-26T14:31:11,942 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor testMonitor 2024-11-26T14:31:11,960 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36307 is added to blk_1073741839_1015 (size=276) 2024-11-26T14:31:11,960 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36277 is added to blk_1073741839_1015 (size=276) 2024-11-26T14:31:11,961 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38147 is added to blk_1073741839_1015 (size=276) 2024-11-26T14:31:11,962 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor testMonitor 2024-11-26T14:31:11,975 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36307 is added to blk_1073741840_1016 (size=230) 2024-11-26T14:31:11,975 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36277 is added to blk_1073741840_1016 (size=230) 2024-11-26T14:31:11,975 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38147 is added to blk_1073741840_1016 (size=230) 2024-11-26T14:31:11,999 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:44999/hbase/WALs/testnameconflictwhensplit0-manual,16010,1732631471861/wal-1, size=276 (276bytes) 2024-11-26T14:31:11,999 DEBUG [Time-limited test {}] util.RecoverLeaseFSUtils(47): Initialize RecoverLeaseFSUtils 2024-11-26T14:31:12,000 DEBUG [Time-limited test {}] util.RecoverLeaseFSUtils(59): set recoverLeaseMethod to org.apache.hadoop.fs.LeaseRecoverable.recoverLease() 2024-11-26T14:31:12,000 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:44999/hbase/WALs/testnameconflictwhensplit0-manual,16010,1732631471861/wal-1 2024-11-26T14:31:12,005 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:44999/hbase/WALs/testnameconflictwhensplit0-manual,16010,1732631471861/wal-1 after 3ms 2024-11-26T14:31:12,011 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:44999/hbase/WALs/testnameconflictwhensplit0-manual,16010,1732631471861/wal-1: isRecoveredEdits=false, hasTagCompression=true, hasValueCompression=false, valueCompressionType=null 2024-11-26T14:31:12,011 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:44999/hbase/WALs/testnameconflictwhensplit0-manual,16010,1732631471861/wal-1 took 14ms 2024-11-26T14:31:12,016 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:44999/hbase/WALs/testnameconflictwhensplit0-manual,16010,1732631471861/wal-1 so closing down 2024-11-26T14:31:12,019 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000001-wal-1.temp 2024-11-26T14:31:12,021 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:44999/hbase/data/default/testReplayEditsWrittenIntoWAL/0646c372d90fdcf989fc1feb0b0b253d/recovered.edits/0000000000000000001-wal-1.temp 2024-11-26T14:31:12,022 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-26T14:31:12,022 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-11-26T14:31:12,031 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36307 is added to blk_1073741841_1017 (size=276) 2024-11-26T14:31:12,032 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36277 is added to blk_1073741841_1017 (size=276) 2024-11-26T14:31:12,033 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38147 is added to blk_1073741841_1017 (size=276) 2024-11-26T14:31:12,035 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:44999/hbase/data/default/testReplayEditsWrittenIntoWAL/0646c372d90fdcf989fc1feb0b0b253d/recovered.edits/0000000000000000001-wal-1.temp (wrote 2 edits, skipped 0 edits in 0 ms) 2024-11-26T14:31:12,038 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:44999/hbase/data/default/testReplayEditsWrittenIntoWAL/0646c372d90fdcf989fc1feb0b0b253d/recovered.edits/0000000000000000001-wal-1.temp to hdfs://localhost:44999/hbase/data/default/testReplayEditsWrittenIntoWAL/0646c372d90fdcf989fc1feb0b0b253d/recovered.edits/0000000000000000002 2024-11-26T14:31:12,041 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 2 edits across 1 Regions in 25 ms; skipped=0; WAL=hdfs://localhost:44999/hbase/WALs/testnameconflictwhensplit0-manual,16010,1732631471861/wal-1, size=276, length=276, corrupted=false, cancelled=false 2024-11-26T14:31:12,041 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:44999/hbase/WALs/testnameconflictwhensplit0-manual,16010,1732631471861/wal-1, journal: Splitting hdfs://localhost:44999/hbase/WALs/testnameconflictwhensplit0-manual,16010,1732631471861/wal-1, size=276 (276bytes) at 1732631471999Finishing writing output for hdfs://localhost:44999/hbase/WALs/testnameconflictwhensplit0-manual,16010,1732631471861/wal-1 so closing down at 1732631472016 (+17 ms)Creating recovered edits writer path=hdfs://localhost:44999/hbase/data/default/testReplayEditsWrittenIntoWAL/0646c372d90fdcf989fc1feb0b0b253d/recovered.edits/0000000000000000001-wal-1.temp at 1732631472021 (+5 ms)3 split writer threads finished at 1732631472022 (+1 ms)Closed recovered edits writer path=hdfs://localhost:44999/hbase/data/default/testReplayEditsWrittenIntoWAL/0646c372d90fdcf989fc1feb0b0b253d/recovered.edits/0000000000000000001-wal-1.temp (wrote 2 edits, skipped 0 edits in 0 ms) at 1732631472035 (+13 ms)Rename recovered edits hdfs://localhost:44999/hbase/data/default/testReplayEditsWrittenIntoWAL/0646c372d90fdcf989fc1feb0b0b253d/recovered.edits/0000000000000000001-wal-1.temp to hdfs://localhost:44999/hbase/data/default/testReplayEditsWrittenIntoWAL/0646c372d90fdcf989fc1feb0b0b253d/recovered.edits/0000000000000000002 at 1732631472038 (+3 ms)Processed 2 edits across 1 Regions in 25 ms; skipped=0; WAL=hdfs://localhost:44999/hbase/WALs/testnameconflictwhensplit0-manual,16010,1732631471861/wal-1, size=276, length=276, corrupted=false, cancelled=false at 1732631472041 (+3 ms) 2024-11-26T14:31:12,056 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:44999/hbase/WALs/testnameconflictwhensplit0-manual,16010,1732631471861/wal-2, size=230 (230bytes) 2024-11-26T14:31:12,056 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:44999/hbase/WALs/testnameconflictwhensplit0-manual,16010,1732631471861/wal-2 2024-11-26T14:31:12,057 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:44999/hbase/WALs/testnameconflictwhensplit0-manual,16010,1732631471861/wal-2 after 1ms 2024-11-26T14:31:12,061 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:44999/hbase/WALs/testnameconflictwhensplit0-manual,16010,1732631471861/wal-2: isRecoveredEdits=false, hasTagCompression=true, hasValueCompression=false, valueCompressionType=null 2024-11-26T14:31:12,061 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:44999/hbase/WALs/testnameconflictwhensplit0-manual,16010,1732631471861/wal-2 took 6ms 2024-11-26T14:31:12,063 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:44999/hbase/WALs/testnameconflictwhensplit0-manual,16010,1732631471861/wal-2 so closing down 2024-11-26T14:31:12,064 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-26T14:31:12,067 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000002-wal-2.temp 2024-11-26T14:31:12,068 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:44999/hbase/data/default/testReplayEditsWrittenIntoWAL/0646c372d90fdcf989fc1feb0b0b253d/recovered.edits/0000000000000000002-wal-2.temp 2024-11-26T14:31:12,069 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-11-26T14:31:12,076 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38147 is added to blk_1073741842_1018 (size=230) 2024-11-26T14:31:12,076 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36277 is added to blk_1073741842_1018 (size=230) 2024-11-26T14:31:12,076 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36307 is added to blk_1073741842_1018 (size=230) 2024-11-26T14:31:12,077 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:44999/hbase/data/default/testReplayEditsWrittenIntoWAL/0646c372d90fdcf989fc1feb0b0b253d/recovered.edits/0000000000000000002-wal-2.temp (wrote 1 edits, skipped 0 edits in 0 ms) 2024-11-26T14:31:12,081 DEBUG [split-log-closeStream-pool-0 {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:44999/hbase/data/default/testReplayEditsWrittenIntoWAL/0646c372d90fdcf989fc1feb0b0b253d/recovered.edits/0000000000000000002: isRecoveredEdits=true, hasTagCompression=true, hasValueCompression=false, valueCompressionType=null 2024-11-26T14:31:12,083 WARN [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(184): Found existing old edits file and we have less entries. Deleting hdfs://localhost:44999/hbase/data/default/testReplayEditsWrittenIntoWAL/0646c372d90fdcf989fc1feb0b0b253d/recovered.edits/0000000000000000002-wal-2.temp, length=230 2024-11-26T14:31:12,085 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 1 edits across 1 Regions in 24 ms; skipped=0; WAL=hdfs://localhost:44999/hbase/WALs/testnameconflictwhensplit0-manual,16010,1732631471861/wal-2, size=230, length=230, corrupted=false, cancelled=false 2024-11-26T14:31:12,085 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:44999/hbase/WALs/testnameconflictwhensplit0-manual,16010,1732631471861/wal-2, journal: Splitting hdfs://localhost:44999/hbase/WALs/testnameconflictwhensplit0-manual,16010,1732631471861/wal-2, size=230 (230bytes) at 1732631472056Finishing writing output for hdfs://localhost:44999/hbase/WALs/testnameconflictwhensplit0-manual,16010,1732631471861/wal-2 so closing down at 1732631472064 (+8 ms)Creating recovered edits writer path=hdfs://localhost:44999/hbase/data/default/testReplayEditsWrittenIntoWAL/0646c372d90fdcf989fc1feb0b0b253d/recovered.edits/0000000000000000002-wal-2.temp at 1732631472068 (+4 ms)3 split writer threads finished at 1732631472069 (+1 ms)Closed recovered edits writer path=hdfs://localhost:44999/hbase/data/default/testReplayEditsWrittenIntoWAL/0646c372d90fdcf989fc1feb0b0b253d/recovered.edits/0000000000000000002-wal-2.temp (wrote 1 edits, skipped 0 edits in 0 ms) at 1732631472077 (+8 ms)Processed 1 edits across 1 Regions in 24 ms; skipped=0; WAL=hdfs://localhost:44999/hbase/WALs/testnameconflictwhensplit0-manual,16010,1732631471861/wal-2, size=230, length=230, corrupted=false, cancelled=false at 1732631472085 (+8 ms) 2024-11-26T14:31:12,085 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-26T14:31:12,088 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:44999/hbase/WALs/testnameconflictwhensplit0-manual,16010,1732631471861, archiveDir=hdfs://localhost:44999/hbase/oldWALs, maxLogs=32 2024-11-26T14:31:12,101 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testnameconflictwhensplit0-manual,16010,1732631471861/wal.1732631472089, exclude list is [], retry=0 2024-11-26T14:31:12,105 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36277,DS-9cb9383d-a6c8-4ba6-83b3-6706331a6539,DISK] 2024-11-26T14:31:12,106 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38147,DS-44b12d44-38d0-4c76-b56f-3ca8d8caa398,DISK] 2024-11-26T14:31:12,106 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36307,DS-dd37de78-b3d5-4d7c-8223-4b15101c21bf,DISK] 2024-11-26T14:31:12,108 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testnameconflictwhensplit0-manual,16010,1732631471861/wal.1732631472089 2024-11-26T14:31:12,109 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:40449:40449),(127.0.0.1/127.0.0.1:35081:35081),(127.0.0.1/127.0.0.1:34431:34431)] 2024-11-26T14:31:12,109 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 0646c372d90fdcf989fc1feb0b0b253d, NAME => 'testReplayEditsWrittenIntoWAL,,1732631471863.0646c372d90fdcf989fc1feb0b0b253d.', STARTKEY => '', ENDKEY => ''} 2024-11-26T14:31:12,109 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenIntoWAL,,1732631471863.0646c372d90fdcf989fc1feb0b0b253d.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-26T14:31:12,109 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 0646c372d90fdcf989fc1feb0b0b253d 2024-11-26T14:31:12,109 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 0646c372d90fdcf989fc1feb0b0b253d 2024-11-26T14:31:12,112 INFO [StoreOpener-0646c372d90fdcf989fc1feb0b0b253d-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 0646c372d90fdcf989fc1feb0b0b253d 2024-11-26T14:31:12,113 INFO [StoreOpener-0646c372d90fdcf989fc1feb0b0b253d-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 0646c372d90fdcf989fc1feb0b0b253d columnFamilyName a 2024-11-26T14:31:12,114 DEBUG [StoreOpener-0646c372d90fdcf989fc1feb0b0b253d-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:31:12,115 INFO [StoreOpener-0646c372d90fdcf989fc1feb0b0b253d-1 {}] regionserver.HStore(327): Store=0646c372d90fdcf989fc1feb0b0b253d/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-26T14:31:12,115 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 0646c372d90fdcf989fc1feb0b0b253d 2024-11-26T14:31:12,116 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44999/hbase/data/default/testReplayEditsWrittenIntoWAL/0646c372d90fdcf989fc1feb0b0b253d 2024-11-26T14:31:12,120 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:44999/hbase/data/default/testReplayEditsWrittenIntoWAL/0646c372d90fdcf989fc1feb0b0b253d 2024-11-26T14:31:12,122 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:44999/hbase/data/default/testReplayEditsWrittenIntoWAL/0646c372d90fdcf989fc1feb0b0b253d/recovered.edits/0000000000000000002 2024-11-26T14:31:12,126 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:44999/hbase/data/default/testReplayEditsWrittenIntoWAL/0646c372d90fdcf989fc1feb0b0b253d/recovered.edits/0000000000000000002: isRecoveredEdits=true, hasTagCompression=true, hasValueCompression=false, valueCompressionType=null 2024-11-26T14:31:12,133 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 2, skipped 0, firstSequenceIdInLog=1, maxSequenceIdInLog=2, path=hdfs://localhost:44999/hbase/data/default/testReplayEditsWrittenIntoWAL/0646c372d90fdcf989fc1feb0b0b253d/recovered.edits/0000000000000000002 2024-11-26T14:31:12,136 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 0646c372d90fdcf989fc1feb0b0b253d 1/1 column families, dataSize=108 B heapSize=512 B 2024-11-26T14:31:12,186 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44999/hbase/data/default/testReplayEditsWrittenIntoWAL/0646c372d90fdcf989fc1feb0b0b253d/.tmp/a/16a5f03881084a30bb9fba532a176766 is 58, key is testReplayEditsWrittenIntoWAL/a:1/1732631471940/Put/seqid=0 2024-11-26T14:31:12,200 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38147 is added to blk_1073741844_1020 (size=5170) 2024-11-26T14:31:12,200 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36277 is added to blk_1073741844_1020 (size=5170) 2024-11-26T14:31:12,200 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36307 is added to blk_1073741844_1020 (size=5170) 2024-11-26T14:31:12,201 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=108 B at sequenceid=2 (bloomFilter=true), to=hdfs://localhost:44999/hbase/data/default/testReplayEditsWrittenIntoWAL/0646c372d90fdcf989fc1feb0b0b253d/.tmp/a/16a5f03881084a30bb9fba532a176766 2024-11-26T14:31:12,248 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44999/hbase/data/default/testReplayEditsWrittenIntoWAL/0646c372d90fdcf989fc1feb0b0b253d/.tmp/a/16a5f03881084a30bb9fba532a176766 as hdfs://localhost:44999/hbase/data/default/testReplayEditsWrittenIntoWAL/0646c372d90fdcf989fc1feb0b0b253d/a/16a5f03881084a30bb9fba532a176766 2024-11-26T14:31:12,258 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44999/hbase/data/default/testReplayEditsWrittenIntoWAL/0646c372d90fdcf989fc1feb0b0b253d/a/16a5f03881084a30bb9fba532a176766, entries=2, sequenceid=2, filesize=5.0 K 2024-11-26T14:31:12,263 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~108 B/108, heapSize ~496 B/496, currentSize=0 B/0 for 0646c372d90fdcf989fc1feb0b0b253d in 126ms, sequenceid=2, compaction requested=false; wal=null 2024-11-26T14:31:12,265 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:44999/hbase/data/default/testReplayEditsWrittenIntoWAL/0646c372d90fdcf989fc1feb0b0b253d/recovered.edits/0000000000000000002 2024-11-26T14:31:12,265 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 0646c372d90fdcf989fc1feb0b0b253d 2024-11-26T14:31:12,266 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 0646c372d90fdcf989fc1feb0b0b253d 2024-11-26T14:31:12,269 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 0646c372d90fdcf989fc1feb0b0b253d 2024-11-26T14:31:12,272 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44999/hbase/data/default/testReplayEditsWrittenIntoWAL/0646c372d90fdcf989fc1feb0b0b253d/recovered.edits/2.seqid, newMaxSeqId=2, maxSeqId=1 2024-11-26T14:31:12,274 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 0646c372d90fdcf989fc1feb0b0b253d; next sequenceid=3; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=70531554, jitterRate=0.051002055406570435}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-26T14:31:12,275 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 0646c372d90fdcf989fc1feb0b0b253d: Writing region info on filesystem at 1732631472110Initializing all the Stores at 1732631472111 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732631472111Obtaining lock to block concurrent updates at 1732631472137 (+26 ms)Preparing flush snapshotting stores in 0646c372d90fdcf989fc1feb0b0b253d at 1732631472137Finished memstore snapshotting testReplayEditsWrittenIntoWAL,,1732631471863.0646c372d90fdcf989fc1feb0b0b253d., syncing WAL and waiting on mvcc, flushsize=dataSize=108, getHeapSize=496, getOffHeapSize=0, getCellsCount=2 at 1732631472139 (+2 ms)Flushing stores of testReplayEditsWrittenIntoWAL,,1732631471863.0646c372d90fdcf989fc1feb0b0b253d. at 1732631472139Flushing 0646c372d90fdcf989fc1feb0b0b253d/a: creating writer at 1732631472140 (+1 ms)Flushing 0646c372d90fdcf989fc1feb0b0b253d/a: appending metadata at 1732631472174 (+34 ms)Flushing 0646c372d90fdcf989fc1feb0b0b253d/a: closing flushed file at 1732631472177 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@3d18861c: reopening flushed file at 1732631472246 (+69 ms)Finished flush of dataSize ~108 B/108, heapSize ~496 B/496, currentSize=0 B/0 for 0646c372d90fdcf989fc1feb0b0b253d in 126ms, sequenceid=2, compaction requested=false; wal=null at 1732631472263 (+17 ms)Cleaning up temporary data from old regions at 1732631472266 (+3 ms)Region opened successfully at 1732631472275 (+9 ms) 2024-11-26T14:31:12,301 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestAsyncWALReplayCompressed#testNameConflictWhenSplit0 Thread=378 (was 367) Potentially hanging thread: PacketResponder: BP-1347678837-172.17.0.3-1732631464290:blk_1073741843_1019, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-1347678837-172.17.0.3-1732631464290:blk_1073741843_1019, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_2043263076_22 at /127.0.0.1:40176 [Waiting for operation #6] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_2043263076_22 at /127.0.0.1:42794 [Receiving block BP-1347678837-172.17.0.3-1732631464290:blk_1073741843_1019] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_2043263076_22 at /127.0.0.1:46942 [Receiving block BP-1347678837-172.17.0.3-1732631464290:blk_1073741843_1019] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-8-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_2043263076_22 at /127.0.0.1:39698 [Receiving block BP-1347678837-172.17.0.3-1732631464290:blk_1073741843_1019] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-0-hdfs://localhost:44999/hbase-prefix:default java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: TestAsyncWALReplay-pool-0 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:62) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:883) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-8-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_2043263076_22 at /127.0.0.1:34130 [Waiting for operation #14] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-8-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-1347678837-172.17.0.3-1732631464290:blk_1073741843_1019, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=703 (was 611) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=224 (was 224), ProcessCount=11 (was 11), AvailableMemoryMB=9456 (was 9469) 2024-11-26T14:31:12,311 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestAsyncWALReplayCompressed#testNameConflictWhenSplit1 Thread=378, OpenFileDescriptor=703, MaxFileDescriptor=1048576, SystemLoadAverage=224, ProcessCount=11, AvailableMemoryMB=9455 2024-11-26T14:31:12,326 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-26T14:31:12,328 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-26T14:31:12,329 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-26T14:31:12,333 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-22454467, suffix=, logDir=hdfs://localhost:44999/hbase/WALs/hregion-22454467, archiveDir=hdfs://localhost:44999/hbase/oldWALs, maxLogs=32 2024-11-26T14:31:12,346 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-22454467/hregion-22454467.1732631472334, exclude list is [], retry=0 2024-11-26T14:31:12,350 DEBUG [AsyncFSWAL-10-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36307,DS-dd37de78-b3d5-4d7c-8223-4b15101c21bf,DISK] 2024-11-26T14:31:12,350 DEBUG [AsyncFSWAL-10-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36277,DS-9cb9383d-a6c8-4ba6-83b3-6706331a6539,DISK] 2024-11-26T14:31:12,350 DEBUG [AsyncFSWAL-10-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38147,DS-44b12d44-38d0-4c76-b56f-3ca8d8caa398,DISK] 2024-11-26T14:31:12,353 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-22454467/hregion-22454467.1732631472334 2024-11-26T14:31:12,353 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:34431:34431),(127.0.0.1/127.0.0.1:40449:40449),(127.0.0.1/127.0.0.1:35081:35081)] 2024-11-26T14:31:12,354 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => 94084a4a2eb835ba68d39d5e4ace1b14, NAME => 'testReplayEditsWrittenIntoWAL,,1732631472326.94084a4a2eb835ba68d39d5e4ace1b14.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testReplayEditsWrittenIntoWAL', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:44999/hbase 2024-11-26T14:31:12,365 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38147 is added to blk_1073741846_1022 (size=64) 2024-11-26T14:31:12,366 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36307 is added to blk_1073741846_1022 (size=64) 2024-11-26T14:31:12,366 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36277 is added to blk_1073741846_1022 (size=64) 2024-11-26T14:31:12,367 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenIntoWAL,,1732631472326.94084a4a2eb835ba68d39d5e4ace1b14.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-26T14:31:12,369 INFO [StoreOpener-94084a4a2eb835ba68d39d5e4ace1b14-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 94084a4a2eb835ba68d39d5e4ace1b14 2024-11-26T14:31:12,371 INFO [StoreOpener-94084a4a2eb835ba68d39d5e4ace1b14-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 94084a4a2eb835ba68d39d5e4ace1b14 columnFamilyName a 2024-11-26T14:31:12,371 DEBUG [StoreOpener-94084a4a2eb835ba68d39d5e4ace1b14-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:31:12,372 INFO [StoreOpener-94084a4a2eb835ba68d39d5e4ace1b14-1 {}] regionserver.HStore(327): Store=94084a4a2eb835ba68d39d5e4ace1b14/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-26T14:31:12,373 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 94084a4a2eb835ba68d39d5e4ace1b14 2024-11-26T14:31:12,374 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44999/hbase/data/default/testReplayEditsWrittenIntoWAL/94084a4a2eb835ba68d39d5e4ace1b14 2024-11-26T14:31:12,375 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44999/hbase/data/default/testReplayEditsWrittenIntoWAL/94084a4a2eb835ba68d39d5e4ace1b14 2024-11-26T14:31:12,375 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 94084a4a2eb835ba68d39d5e4ace1b14 2024-11-26T14:31:12,375 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 94084a4a2eb835ba68d39d5e4ace1b14 2024-11-26T14:31:12,378 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 94084a4a2eb835ba68d39d5e4ace1b14 2024-11-26T14:31:12,382 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44999/hbase/data/default/testReplayEditsWrittenIntoWAL/94084a4a2eb835ba68d39d5e4ace1b14/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-26T14:31:12,383 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 94084a4a2eb835ba68d39d5e4ace1b14; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=61806323, jitterRate=-0.07901401817798615}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-26T14:31:12,383 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 94084a4a2eb835ba68d39d5e4ace1b14: Writing region info on filesystem at 1732631472367Initializing all the Stores at 1732631472368 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732631472369 (+1 ms)Cleaning up temporary data from old regions at 1732631472375 (+6 ms)Region opened successfully at 1732631472383 (+8 ms) 2024-11-26T14:31:12,383 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 94084a4a2eb835ba68d39d5e4ace1b14, disabling compactions & flushes 2024-11-26T14:31:12,383 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsWrittenIntoWAL,,1732631472326.94084a4a2eb835ba68d39d5e4ace1b14. 2024-11-26T14:31:12,383 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsWrittenIntoWAL,,1732631472326.94084a4a2eb835ba68d39d5e4ace1b14. 2024-11-26T14:31:12,383 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsWrittenIntoWAL,,1732631472326.94084a4a2eb835ba68d39d5e4ace1b14. after waiting 0 ms 2024-11-26T14:31:12,383 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsWrittenIntoWAL,,1732631472326.94084a4a2eb835ba68d39d5e4ace1b14. 2024-11-26T14:31:12,384 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsWrittenIntoWAL,,1732631472326.94084a4a2eb835ba68d39d5e4ace1b14. 2024-11-26T14:31:12,384 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 94084a4a2eb835ba68d39d5e4ace1b14: Waiting for close lock at 1732631472383Disabling compacts and flushes for region at 1732631472383Disabling writes for close at 1732631472383Writing region close event to WAL at 1732631472384 (+1 ms)Closed at 1732631472384 2024-11-26T14:31:12,388 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38147 is added to blk_1073741845_1021 (size=93) 2024-11-26T14:31:12,388 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36277 is added to blk_1073741845_1021 (size=93) 2024-11-26T14:31:12,388 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36307 is added to blk_1073741845_1021 (size=93) 2024-11-26T14:31:12,391 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-11-26T14:31:12,391 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-22454467:(num 1732631472334) 2024-11-26T14:31:12,392 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor testMonitor 2024-11-26T14:31:12,403 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38147 is added to blk_1073741847_1023 (size=276) 2024-11-26T14:31:12,403 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36307 is added to blk_1073741847_1023 (size=276) 2024-11-26T14:31:12,404 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36277 is added to blk_1073741847_1023 (size=276) 2024-11-26T14:31:12,405 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor testMonitor 2024-11-26T14:31:12,416 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36307 is added to blk_1073741848_1024 (size=230) 2024-11-26T14:31:12,416 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36277 is added to blk_1073741848_1024 (size=230) 2024-11-26T14:31:12,416 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38147 is added to blk_1073741848_1024 (size=230) 2024-11-26T14:31:12,432 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:44999/hbase/WALs/testnameconflictwhensplit1-manual,16010,1732631472325/wal-2, size=230 (230bytes) 2024-11-26T14:31:12,432 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:44999/hbase/WALs/testnameconflictwhensplit1-manual,16010,1732631472325/wal-2 2024-11-26T14:31:12,433 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:44999/hbase/WALs/testnameconflictwhensplit1-manual,16010,1732631472325/wal-2 after 1ms 2024-11-26T14:31:12,437 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:44999/hbase/WALs/testnameconflictwhensplit1-manual,16010,1732631472325/wal-2: isRecoveredEdits=false, hasTagCompression=true, hasValueCompression=false, valueCompressionType=null 2024-11-26T14:31:12,438 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:44999/hbase/WALs/testnameconflictwhensplit1-manual,16010,1732631472325/wal-2 took 7ms 2024-11-26T14:31:12,441 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:44999/hbase/WALs/testnameconflictwhensplit1-manual,16010,1732631472325/wal-2 so closing down 2024-11-26T14:31:12,441 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-26T14:31:12,443 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000002-wal-2.temp 2024-11-26T14:31:12,445 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:44999/hbase/data/default/testReplayEditsWrittenIntoWAL/94084a4a2eb835ba68d39d5e4ace1b14/recovered.edits/0000000000000000002-wal-2.temp 2024-11-26T14:31:12,445 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-11-26T14:31:12,453 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36277 is added to blk_1073741849_1025 (size=230) 2024-11-26T14:31:12,453 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38147 is added to blk_1073741849_1025 (size=230) 2024-11-26T14:31:12,454 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36307 is added to blk_1073741849_1025 (size=230) 2024-11-26T14:31:12,454 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:44999/hbase/data/default/testReplayEditsWrittenIntoWAL/94084a4a2eb835ba68d39d5e4ace1b14/recovered.edits/0000000000000000002-wal-2.temp (wrote 1 edits, skipped 0 edits in 0 ms) 2024-11-26T14:31:12,456 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:44999/hbase/data/default/testReplayEditsWrittenIntoWAL/94084a4a2eb835ba68d39d5e4ace1b14/recovered.edits/0000000000000000002-wal-2.temp to hdfs://localhost:44999/hbase/data/default/testReplayEditsWrittenIntoWAL/94084a4a2eb835ba68d39d5e4ace1b14/recovered.edits/0000000000000000002 2024-11-26T14:31:12,457 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 1 edits across 1 Regions in 18 ms; skipped=0; WAL=hdfs://localhost:44999/hbase/WALs/testnameconflictwhensplit1-manual,16010,1732631472325/wal-2, size=230, length=230, corrupted=false, cancelled=false 2024-11-26T14:31:12,457 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:44999/hbase/WALs/testnameconflictwhensplit1-manual,16010,1732631472325/wal-2, journal: Splitting hdfs://localhost:44999/hbase/WALs/testnameconflictwhensplit1-manual,16010,1732631472325/wal-2, size=230 (230bytes) at 1732631472432Finishing writing output for hdfs://localhost:44999/hbase/WALs/testnameconflictwhensplit1-manual,16010,1732631472325/wal-2 so closing down at 1732631472441 (+9 ms)Creating recovered edits writer path=hdfs://localhost:44999/hbase/data/default/testReplayEditsWrittenIntoWAL/94084a4a2eb835ba68d39d5e4ace1b14/recovered.edits/0000000000000000002-wal-2.temp at 1732631472445 (+4 ms)3 split writer threads finished at 1732631472445Closed recovered edits writer path=hdfs://localhost:44999/hbase/data/default/testReplayEditsWrittenIntoWAL/94084a4a2eb835ba68d39d5e4ace1b14/recovered.edits/0000000000000000002-wal-2.temp (wrote 1 edits, skipped 0 edits in 0 ms) at 1732631472454 (+9 ms)Rename recovered edits hdfs://localhost:44999/hbase/data/default/testReplayEditsWrittenIntoWAL/94084a4a2eb835ba68d39d5e4ace1b14/recovered.edits/0000000000000000002-wal-2.temp to hdfs://localhost:44999/hbase/data/default/testReplayEditsWrittenIntoWAL/94084a4a2eb835ba68d39d5e4ace1b14/recovered.edits/0000000000000000002 at 1732631472456 (+2 ms)Processed 1 edits across 1 Regions in 18 ms; skipped=0; WAL=hdfs://localhost:44999/hbase/WALs/testnameconflictwhensplit1-manual,16010,1732631472325/wal-2, size=230, length=230, corrupted=false, cancelled=false at 1732631472457 (+1 ms) 2024-11-26T14:31:12,471 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:44999/hbase/WALs/testnameconflictwhensplit1-manual,16010,1732631472325/wal-1, size=276 (276bytes) 2024-11-26T14:31:12,471 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:44999/hbase/WALs/testnameconflictwhensplit1-manual,16010,1732631472325/wal-1 2024-11-26T14:31:12,472 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:44999/hbase/WALs/testnameconflictwhensplit1-manual,16010,1732631472325/wal-1 after 1ms 2024-11-26T14:31:12,476 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:44999/hbase/WALs/testnameconflictwhensplit1-manual,16010,1732631472325/wal-1: isRecoveredEdits=false, hasTagCompression=true, hasValueCompression=false, valueCompressionType=null 2024-11-26T14:31:12,476 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:44999/hbase/WALs/testnameconflictwhensplit1-manual,16010,1732631472325/wal-1 took 5ms 2024-11-26T14:31:12,480 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:44999/hbase/WALs/testnameconflictwhensplit1-manual,16010,1732631472325/wal-1 so closing down 2024-11-26T14:31:12,480 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-26T14:31:12,482 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000001-wal-1.temp 2024-11-26T14:31:12,484 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:44999/hbase/data/default/testReplayEditsWrittenIntoWAL/94084a4a2eb835ba68d39d5e4ace1b14/recovered.edits/0000000000000000001-wal-1.temp 2024-11-26T14:31:12,484 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-11-26T14:31:12,491 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36307 is added to blk_1073741850_1026 (size=276) 2024-11-26T14:31:12,492 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38147 is added to blk_1073741850_1026 (size=276) 2024-11-26T14:31:12,492 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36277 is added to blk_1073741850_1026 (size=276) 2024-11-26T14:31:12,493 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:44999/hbase/data/default/testReplayEditsWrittenIntoWAL/94084a4a2eb835ba68d39d5e4ace1b14/recovered.edits/0000000000000000001-wal-1.temp (wrote 2 edits, skipped 0 edits in 0 ms) 2024-11-26T14:31:12,496 DEBUG [split-log-closeStream-pool-0 {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:44999/hbase/data/default/testReplayEditsWrittenIntoWAL/94084a4a2eb835ba68d39d5e4ace1b14/recovered.edits/0000000000000000002: isRecoveredEdits=true, hasTagCompression=true, hasValueCompression=false, valueCompressionType=null 2024-11-26T14:31:12,499 WARN [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(175): Found existing old edits file. It could be the result of a previous failed split attempt or we have duplicated wal entries. Deleting hdfs://localhost:44999/hbase/data/default/testReplayEditsWrittenIntoWAL/94084a4a2eb835ba68d39d5e4ace1b14/recovered.edits/0000000000000000002, length=230 2024-11-26T14:31:12,501 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:44999/hbase/data/default/testReplayEditsWrittenIntoWAL/94084a4a2eb835ba68d39d5e4ace1b14/recovered.edits/0000000000000000001-wal-1.temp to hdfs://localhost:44999/hbase/data/default/testReplayEditsWrittenIntoWAL/94084a4a2eb835ba68d39d5e4ace1b14/recovered.edits/0000000000000000002 2024-11-26T14:31:12,501 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 2 edits across 1 Regions in 24 ms; skipped=0; WAL=hdfs://localhost:44999/hbase/WALs/testnameconflictwhensplit1-manual,16010,1732631472325/wal-1, size=276, length=276, corrupted=false, cancelled=false 2024-11-26T14:31:12,502 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:44999/hbase/WALs/testnameconflictwhensplit1-manual,16010,1732631472325/wal-1, journal: Splitting hdfs://localhost:44999/hbase/WALs/testnameconflictwhensplit1-manual,16010,1732631472325/wal-1, size=276 (276bytes) at 1732631472471Finishing writing output for hdfs://localhost:44999/hbase/WALs/testnameconflictwhensplit1-manual,16010,1732631472325/wal-1 so closing down at 1732631472480 (+9 ms)Creating recovered edits writer path=hdfs://localhost:44999/hbase/data/default/testReplayEditsWrittenIntoWAL/94084a4a2eb835ba68d39d5e4ace1b14/recovered.edits/0000000000000000001-wal-1.temp at 1732631472484 (+4 ms)3 split writer threads finished at 1732631472484Closed recovered edits writer path=hdfs://localhost:44999/hbase/data/default/testReplayEditsWrittenIntoWAL/94084a4a2eb835ba68d39d5e4ace1b14/recovered.edits/0000000000000000001-wal-1.temp (wrote 2 edits, skipped 0 edits in 0 ms) at 1732631472493 (+9 ms)Rename recovered edits hdfs://localhost:44999/hbase/data/default/testReplayEditsWrittenIntoWAL/94084a4a2eb835ba68d39d5e4ace1b14/recovered.edits/0000000000000000001-wal-1.temp to hdfs://localhost:44999/hbase/data/default/testReplayEditsWrittenIntoWAL/94084a4a2eb835ba68d39d5e4ace1b14/recovered.edits/0000000000000000002 at 1732631472501 (+8 ms)Processed 2 edits across 1 Regions in 24 ms; skipped=0; WAL=hdfs://localhost:44999/hbase/WALs/testnameconflictwhensplit1-manual,16010,1732631472325/wal-1, size=276, length=276, corrupted=false, cancelled=false at 1732631472501 2024-11-26T14:31:12,502 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-26T14:31:12,504 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:44999/hbase/WALs/testnameconflictwhensplit1-manual,16010,1732631472325, archiveDir=hdfs://localhost:44999/hbase/oldWALs, maxLogs=32 2024-11-26T14:31:12,517 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testnameconflictwhensplit1-manual,16010,1732631472325/wal.1732631472505, exclude list is [], retry=0 2024-11-26T14:31:12,521 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36307,DS-dd37de78-b3d5-4d7c-8223-4b15101c21bf,DISK] 2024-11-26T14:31:12,522 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36277,DS-9cb9383d-a6c8-4ba6-83b3-6706331a6539,DISK] 2024-11-26T14:31:12,522 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38147,DS-44b12d44-38d0-4c76-b56f-3ca8d8caa398,DISK] 2024-11-26T14:31:12,525 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testnameconflictwhensplit1-manual,16010,1732631472325/wal.1732631472505 2024-11-26T14:31:12,526 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:34431:34431),(127.0.0.1/127.0.0.1:40449:40449),(127.0.0.1/127.0.0.1:35081:35081)] 2024-11-26T14:31:12,526 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 94084a4a2eb835ba68d39d5e4ace1b14, NAME => 'testReplayEditsWrittenIntoWAL,,1732631472326.94084a4a2eb835ba68d39d5e4ace1b14.', STARTKEY => '', ENDKEY => ''} 2024-11-26T14:31:12,526 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenIntoWAL,,1732631472326.94084a4a2eb835ba68d39d5e4ace1b14.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-26T14:31:12,526 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 94084a4a2eb835ba68d39d5e4ace1b14 2024-11-26T14:31:12,526 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 94084a4a2eb835ba68d39d5e4ace1b14 2024-11-26T14:31:12,529 INFO [StoreOpener-94084a4a2eb835ba68d39d5e4ace1b14-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 94084a4a2eb835ba68d39d5e4ace1b14 2024-11-26T14:31:12,530 INFO [StoreOpener-94084a4a2eb835ba68d39d5e4ace1b14-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 94084a4a2eb835ba68d39d5e4ace1b14 columnFamilyName a 2024-11-26T14:31:12,531 DEBUG [StoreOpener-94084a4a2eb835ba68d39d5e4ace1b14-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:31:12,531 INFO [StoreOpener-94084a4a2eb835ba68d39d5e4ace1b14-1 {}] regionserver.HStore(327): Store=94084a4a2eb835ba68d39d5e4ace1b14/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-26T14:31:12,532 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 94084a4a2eb835ba68d39d5e4ace1b14 2024-11-26T14:31:12,533 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44999/hbase/data/default/testReplayEditsWrittenIntoWAL/94084a4a2eb835ba68d39d5e4ace1b14 2024-11-26T14:31:12,536 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:44999/hbase/data/default/testReplayEditsWrittenIntoWAL/94084a4a2eb835ba68d39d5e4ace1b14 2024-11-26T14:31:12,537 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:44999/hbase/data/default/testReplayEditsWrittenIntoWAL/94084a4a2eb835ba68d39d5e4ace1b14/recovered.edits/0000000000000000002 2024-11-26T14:31:12,540 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:44999/hbase/data/default/testReplayEditsWrittenIntoWAL/94084a4a2eb835ba68d39d5e4ace1b14/recovered.edits/0000000000000000002: isRecoveredEdits=true, hasTagCompression=true, hasValueCompression=false, valueCompressionType=null 2024-11-26T14:31:12,542 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 2, skipped 0, firstSequenceIdInLog=1, maxSequenceIdInLog=2, path=hdfs://localhost:44999/hbase/data/default/testReplayEditsWrittenIntoWAL/94084a4a2eb835ba68d39d5e4ace1b14/recovered.edits/0000000000000000002 2024-11-26T14:31:12,542 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 94084a4a2eb835ba68d39d5e4ace1b14 1/1 column families, dataSize=108 B heapSize=512 B 2024-11-26T14:31:12,559 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44999/hbase/data/default/testReplayEditsWrittenIntoWAL/94084a4a2eb835ba68d39d5e4ace1b14/.tmp/a/2e476825de08403fa539c87d1ff633cb is 58, key is testReplayEditsWrittenIntoWAL/a:1/1732631472392/Put/seqid=0 2024-11-26T14:31:12,568 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38147 is added to blk_1073741852_1028 (size=5170) 2024-11-26T14:31:12,569 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36277 is added to blk_1073741852_1028 (size=5170) 2024-11-26T14:31:12,569 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36307 is added to blk_1073741852_1028 (size=5170) 2024-11-26T14:31:12,570 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=108 B at sequenceid=2 (bloomFilter=true), to=hdfs://localhost:44999/hbase/data/default/testReplayEditsWrittenIntoWAL/94084a4a2eb835ba68d39d5e4ace1b14/.tmp/a/2e476825de08403fa539c87d1ff633cb 2024-11-26T14:31:12,581 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44999/hbase/data/default/testReplayEditsWrittenIntoWAL/94084a4a2eb835ba68d39d5e4ace1b14/.tmp/a/2e476825de08403fa539c87d1ff633cb as hdfs://localhost:44999/hbase/data/default/testReplayEditsWrittenIntoWAL/94084a4a2eb835ba68d39d5e4ace1b14/a/2e476825de08403fa539c87d1ff633cb 2024-11-26T14:31:12,591 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44999/hbase/data/default/testReplayEditsWrittenIntoWAL/94084a4a2eb835ba68d39d5e4ace1b14/a/2e476825de08403fa539c87d1ff633cb, entries=2, sequenceid=2, filesize=5.0 K 2024-11-26T14:31:12,592 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~108 B/108, heapSize ~496 B/496, currentSize=0 B/0 for 94084a4a2eb835ba68d39d5e4ace1b14 in 50ms, sequenceid=2, compaction requested=false; wal=null 2024-11-26T14:31:12,593 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:44999/hbase/data/default/testReplayEditsWrittenIntoWAL/94084a4a2eb835ba68d39d5e4ace1b14/recovered.edits/0000000000000000002 2024-11-26T14:31:12,593 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 94084a4a2eb835ba68d39d5e4ace1b14 2024-11-26T14:31:12,593 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 94084a4a2eb835ba68d39d5e4ace1b14 2024-11-26T14:31:12,596 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 94084a4a2eb835ba68d39d5e4ace1b14 2024-11-26T14:31:12,599 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44999/hbase/data/default/testReplayEditsWrittenIntoWAL/94084a4a2eb835ba68d39d5e4ace1b14/recovered.edits/2.seqid, newMaxSeqId=2, maxSeqId=1 2024-11-26T14:31:12,601 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 94084a4a2eb835ba68d39d5e4ace1b14; next sequenceid=3; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=68926696, jitterRate=0.027087807655334473}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-26T14:31:12,601 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 94084a4a2eb835ba68d39d5e4ace1b14: Writing region info on filesystem at 1732631472527Initializing all the Stores at 1732631472528 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732631472528Obtaining lock to block concurrent updates at 1732631472542 (+14 ms)Preparing flush snapshotting stores in 94084a4a2eb835ba68d39d5e4ace1b14 at 1732631472542Finished memstore snapshotting testReplayEditsWrittenIntoWAL,,1732631472326.94084a4a2eb835ba68d39d5e4ace1b14., syncing WAL and waiting on mvcc, flushsize=dataSize=108, getHeapSize=496, getOffHeapSize=0, getCellsCount=2 at 1732631472542Flushing stores of testReplayEditsWrittenIntoWAL,,1732631472326.94084a4a2eb835ba68d39d5e4ace1b14. at 1732631472542Flushing 94084a4a2eb835ba68d39d5e4ace1b14/a: creating writer at 1732631472542Flushing 94084a4a2eb835ba68d39d5e4ace1b14/a: appending metadata at 1732631472557 (+15 ms)Flushing 94084a4a2eb835ba68d39d5e4ace1b14/a: closing flushed file at 1732631472558 (+1 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@4168c90a: reopening flushed file at 1732631472579 (+21 ms)Finished flush of dataSize ~108 B/108, heapSize ~496 B/496, currentSize=0 B/0 for 94084a4a2eb835ba68d39d5e4ace1b14 in 50ms, sequenceid=2, compaction requested=false; wal=null at 1732631472592 (+13 ms)Cleaning up temporary data from old regions at 1732631472593 (+1 ms)Region opened successfully at 1732631472601 (+8 ms) 2024-11-26T14:31:12,620 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestAsyncWALReplayCompressed#testNameConflictWhenSplit1 Thread=388 (was 378) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_2043263076_22 at /127.0.0.1:40176 [Waiting for operation #8] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-1347678837-172.17.0.3-1732631464290:blk_1073741851_1027, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-10-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_2043263076_22 at /127.0.0.1:34130 [Waiting for operation #21] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-10-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_2043263076_22 at /127.0.0.1:39756 [Receiving block BP-1347678837-172.17.0.3-1732631464290:blk_1073741851_1027] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-1347678837-172.17.0.3-1732631464290:blk_1073741851_1027, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_2043263076_22 at /127.0.0.1:39804 [Waiting for operation #8] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-1347678837-172.17.0.3-1732631464290:blk_1073741851_1027, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_2043263076_22 at /127.0.0.1:47034 [Receiving block BP-1347678837-172.17.0.3-1732631464290:blk_1073741851_1027] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_2043263076_22 at /127.0.0.1:42866 [Receiving block BP-1347678837-172.17.0.3-1732631464290:blk_1073741851_1027] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-10-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=785 (was 703) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=224 (was 224), ProcessCount=11 (was 11), AvailableMemoryMB=9441 (was 9455) 2024-11-26T14:31:12,632 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestAsyncWALReplayCompressed#testReplayEditsWrittenIntoWAL Thread=388, OpenFileDescriptor=785, MaxFileDescriptor=1048576, SystemLoadAverage=224, ProcessCount=11, AvailableMemoryMB=9435 2024-11-26T14:31:12,648 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-26T14:31:12,650 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-26T14:31:12,651 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-26T14:31:12,654 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-53165116, suffix=, logDir=hdfs://localhost:44999/hbase/WALs/hregion-53165116, archiveDir=hdfs://localhost:44999/hbase/oldWALs, maxLogs=32 2024-11-26T14:31:12,668 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-53165116/hregion-53165116.1732631472655, exclude list is [], retry=0 2024-11-26T14:31:12,671 DEBUG [AsyncFSWAL-12-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36277,DS-9cb9383d-a6c8-4ba6-83b3-6706331a6539,DISK] 2024-11-26T14:31:12,672 DEBUG [AsyncFSWAL-12-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36307,DS-dd37de78-b3d5-4d7c-8223-4b15101c21bf,DISK] 2024-11-26T14:31:12,672 DEBUG [AsyncFSWAL-12-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38147,DS-44b12d44-38d0-4c76-b56f-3ca8d8caa398,DISK] 2024-11-26T14:31:12,675 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-53165116/hregion-53165116.1732631472655 2024-11-26T14:31:12,675 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:40449:40449),(127.0.0.1/127.0.0.1:34431:34431),(127.0.0.1/127.0.0.1:35081:35081)] 2024-11-26T14:31:12,676 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => 6aa243d9781ecdebfdec5139a938dbfa, NAME => 'testReplayEditsWrittenIntoWAL,,1732631472648.6aa243d9781ecdebfdec5139a938dbfa.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testReplayEditsWrittenIntoWAL', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:44999/hbase 2024-11-26T14:31:12,686 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36307 is added to blk_1073741854_1030 (size=64) 2024-11-26T14:31:12,687 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36277 is added to blk_1073741854_1030 (size=64) 2024-11-26T14:31:12,687 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38147 is added to blk_1073741854_1030 (size=64) 2024-11-26T14:31:12,689 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenIntoWAL,,1732631472648.6aa243d9781ecdebfdec5139a938dbfa.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-26T14:31:12,691 INFO [StoreOpener-6aa243d9781ecdebfdec5139a938dbfa-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 6aa243d9781ecdebfdec5139a938dbfa 2024-11-26T14:31:12,693 INFO [StoreOpener-6aa243d9781ecdebfdec5139a938dbfa-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 6aa243d9781ecdebfdec5139a938dbfa columnFamilyName a 2024-11-26T14:31:12,693 DEBUG [StoreOpener-6aa243d9781ecdebfdec5139a938dbfa-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:31:12,694 INFO [StoreOpener-6aa243d9781ecdebfdec5139a938dbfa-1 {}] regionserver.HStore(327): Store=6aa243d9781ecdebfdec5139a938dbfa/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-26T14:31:12,694 INFO [StoreOpener-6aa243d9781ecdebfdec5139a938dbfa-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 6aa243d9781ecdebfdec5139a938dbfa 2024-11-26T14:31:12,696 INFO [StoreOpener-6aa243d9781ecdebfdec5139a938dbfa-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 6aa243d9781ecdebfdec5139a938dbfa columnFamilyName b 2024-11-26T14:31:12,696 DEBUG [StoreOpener-6aa243d9781ecdebfdec5139a938dbfa-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:31:12,697 INFO [StoreOpener-6aa243d9781ecdebfdec5139a938dbfa-1 {}] regionserver.HStore(327): Store=6aa243d9781ecdebfdec5139a938dbfa/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-26T14:31:12,697 INFO [StoreOpener-6aa243d9781ecdebfdec5139a938dbfa-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 6aa243d9781ecdebfdec5139a938dbfa 2024-11-26T14:31:12,699 INFO [StoreOpener-6aa243d9781ecdebfdec5139a938dbfa-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 6aa243d9781ecdebfdec5139a938dbfa columnFamilyName c 2024-11-26T14:31:12,699 DEBUG [StoreOpener-6aa243d9781ecdebfdec5139a938dbfa-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:31:12,699 INFO [StoreOpener-6aa243d9781ecdebfdec5139a938dbfa-1 {}] regionserver.HStore(327): Store=6aa243d9781ecdebfdec5139a938dbfa/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-26T14:31:12,699 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 6aa243d9781ecdebfdec5139a938dbfa 2024-11-26T14:31:12,700 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44999/hbase/data/default/testReplayEditsWrittenIntoWAL/6aa243d9781ecdebfdec5139a938dbfa 2024-11-26T14:31:12,701 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44999/hbase/data/default/testReplayEditsWrittenIntoWAL/6aa243d9781ecdebfdec5139a938dbfa 2024-11-26T14:31:12,702 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 6aa243d9781ecdebfdec5139a938dbfa 2024-11-26T14:31:12,702 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 6aa243d9781ecdebfdec5139a938dbfa 2024-11-26T14:31:12,703 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsWrittenIntoWAL descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-26T14:31:12,704 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 6aa243d9781ecdebfdec5139a938dbfa 2024-11-26T14:31:12,706 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44999/hbase/data/default/testReplayEditsWrittenIntoWAL/6aa243d9781ecdebfdec5139a938dbfa/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-26T14:31:12,707 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 6aa243d9781ecdebfdec5139a938dbfa; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=72053486, jitterRate=0.07368060946464539}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-26T14:31:12,707 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 6aa243d9781ecdebfdec5139a938dbfa: Writing region info on filesystem at 1732631472689Initializing all the Stores at 1732631472691 (+2 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732631472691Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732631472691Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732631472691Cleaning up temporary data from old regions at 1732631472702 (+11 ms)Region opened successfully at 1732631472707 (+5 ms) 2024-11-26T14:31:12,708 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 6aa243d9781ecdebfdec5139a938dbfa, disabling compactions & flushes 2024-11-26T14:31:12,708 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsWrittenIntoWAL,,1732631472648.6aa243d9781ecdebfdec5139a938dbfa. 2024-11-26T14:31:12,708 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsWrittenIntoWAL,,1732631472648.6aa243d9781ecdebfdec5139a938dbfa. 2024-11-26T14:31:12,708 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsWrittenIntoWAL,,1732631472648.6aa243d9781ecdebfdec5139a938dbfa. after waiting 0 ms 2024-11-26T14:31:12,708 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsWrittenIntoWAL,,1732631472648.6aa243d9781ecdebfdec5139a938dbfa. 2024-11-26T14:31:12,708 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsWrittenIntoWAL,,1732631472648.6aa243d9781ecdebfdec5139a938dbfa. 2024-11-26T14:31:12,708 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 6aa243d9781ecdebfdec5139a938dbfa: Waiting for close lock at 1732631472707Disabling compacts and flushes for region at 1732631472707Disabling writes for close at 1732631472708 (+1 ms)Writing region close event to WAL at 1732631472708Closed at 1732631472708 2024-11-26T14:31:12,712 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36307 is added to blk_1073741853_1029 (size=93) 2024-11-26T14:31:12,712 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36277 is added to blk_1073741853_1029 (size=93) 2024-11-26T14:31:12,712 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38147 is added to blk_1073741853_1029 (size=93) 2024-11-26T14:31:12,715 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-11-26T14:31:12,715 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-53165116:(num 1732631472655) 2024-11-26T14:31:12,716 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-26T14:31:12,718 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:44999/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1732631472647, archiveDir=hdfs://localhost:44999/hbase/oldWALs, maxLogs=32 2024-11-26T14:31:12,731 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testreplayeditswrittenintowal-manual,16010,1732631472647/wal.1732631472718, exclude list is [], retry=0 2024-11-26T14:31:12,734 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36277,DS-9cb9383d-a6c8-4ba6-83b3-6706331a6539,DISK] 2024-11-26T14:31:12,735 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38147,DS-44b12d44-38d0-4c76-b56f-3ca8d8caa398,DISK] 2024-11-26T14:31:12,735 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36307,DS-dd37de78-b3d5-4d7c-8223-4b15101c21bf,DISK] 2024-11-26T14:31:12,738 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testreplayeditswrittenintowal-manual,16010,1732631472647/wal.1732631472718 2024-11-26T14:31:12,738 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:40449:40449),(127.0.0.1/127.0.0.1:35081:35081),(127.0.0.1/127.0.0.1:34431:34431)] 2024-11-26T14:31:12,990 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:44999/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1732631472647/wal.1732631472718, size=0 (0bytes) 2024-11-26T14:31:12,990 WARN [Time-limited test {}] wal.WALSplitter(453): File hdfs://localhost:44999/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1732631472647/wal.1732631472718 might be still open, length is 0 2024-11-26T14:31:12,990 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:44999/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1732631472647/wal.1732631472718 2024-11-26T14:31:12,991 WARN [IPC Server handler 2 on default port 44999 {}] namenode.FSNamesystem(3854): DIR* NameSystem.internalReleaseLease: File /hbase/WALs/testreplayeditswrittenintowal-manual,16010,1732631472647/wal.1732631472718 has not been closed. Lease recovery is in progress. RecoveryId = 1032 for block blk_1073741855_1031 2024-11-26T14:31:12,992 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=0 on file=hdfs://localhost:44999/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1732631472647/wal.1732631472718 after 2ms 2024-11-26T14:31:13,866 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_2043263076_22 at /127.0.0.1:47072 [Receiving block BP-1347678837-172.17.0.3-1732631464290:blk_1073741855_1031] {}] datanode.DataXceiver(331): 127.0.0.1:36277:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:47072 dst: /127.0.0.1:36277 java.io.InterruptedIOException: Interrupted while waiting for IO on channel java.nio.channels.SocketChannel[connected local=localhost/127.0.0.1:36277 remote=/127.0.0.1:47072]. Total timeout mills is 60000, 59090 millis timeout left. at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:350) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?] at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?] at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?] at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?] at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?] at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T14:31:13,867 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_2043263076_22 at /127.0.0.1:39792 [Receiving block BP-1347678837-172.17.0.3-1732631464290:blk_1073741855_1031] {}] datanode.DataXceiver(331): 127.0.0.1:36307:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:39792 dst: /127.0.0.1:36307 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T14:31:13,867 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_2043263076_22 at /127.0.0.1:42906 [Receiving block BP-1347678837-172.17.0.3-1732631464290:blk_1073741855_1031] {}] datanode.DataXceiver(331): 127.0.0.1:38147:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:42906 dst: /127.0.0.1:38147 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T14:31:13,875 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36277 is added to blk_1073741855_1032 (size=200589) 2024-11-26T14:31:13,877 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38147 is added to blk_1073741855_1032 (size=200589) 2024-11-26T14:31:13,877 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36307 is added to blk_1073741855_1032 (size=200589) 2024-11-26T14:31:16,993 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=1 on file=hdfs://localhost:44999/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1732631472647/wal.1732631472718 after 4003ms 2024-11-26T14:31:17,001 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:44999/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1732631472647/wal.1732631472718: isRecoveredEdits=false, hasTagCompression=true, hasValueCompression=false, valueCompressionType=null 2024-11-26T14:31:17,003 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:44999/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1732631472647/wal.1732631472718 took 4013ms 2024-11-26T14:31:17,008 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000001-wal.1732631472718.temp 2024-11-26T14:31:17,011 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:44999/hbase/data/default/testReplayEditsWrittenIntoWAL/6aa243d9781ecdebfdec5139a938dbfa/recovered.edits/0000000000000000001-wal.1732631472718.temp 2024-11-26T14:31:17,011 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-26T14:31:17,086 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta' 2024-11-26T14:31:17,131 INFO [Time-limited test {}] wal.WALSplitter(493): EOF from hdfs://localhost:44999/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1732631472647/wal.1732631472718; continuing. 2024-11-26T14:31:17,131 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:44999/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1732631472647/wal.1732631472718 so closing down 2024-11-26T14:31:17,132 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-26T14:31:17,132 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-11-26T14:31:17,134 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36307 is added to blk_1073741856_1033 (size=200597) 2024-11-26T14:31:17,134 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36277 is added to blk_1073741856_1033 (size=200597) 2024-11-26T14:31:17,135 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38147 is added to blk_1073741856_1033 (size=200597) 2024-11-26T14:31:17,135 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:44999/hbase/data/default/testReplayEditsWrittenIntoWAL/6aa243d9781ecdebfdec5139a938dbfa/recovered.edits/0000000000000000001-wal.1732631472718.temp (wrote 3002 edits, skipped 0 edits in 49 ms) 2024-11-26T14:31:17,137 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:44999/hbase/data/default/testReplayEditsWrittenIntoWAL/6aa243d9781ecdebfdec5139a938dbfa/recovered.edits/0000000000000000001-wal.1732631472718.temp to hdfs://localhost:44999/hbase/data/default/testReplayEditsWrittenIntoWAL/6aa243d9781ecdebfdec5139a938dbfa/recovered.edits/0000000000000003002 2024-11-26T14:31:17,137 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 3002 edits across 1 Regions in 133 ms; skipped=0; WAL=hdfs://localhost:44999/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1732631472647/wal.1732631472718, size=0, length=0, corrupted=false, cancelled=false 2024-11-26T14:31:17,138 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:44999/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1732631472647/wal.1732631472718, journal: Splitting hdfs://localhost:44999/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1732631472647/wal.1732631472718, size=0 (0bytes) at 1732631472990Creating recovered edits writer path=hdfs://localhost:44999/hbase/data/default/testReplayEditsWrittenIntoWAL/6aa243d9781ecdebfdec5139a938dbfa/recovered.edits/0000000000000000001-wal.1732631472718.temp at 1732631477011 (+4021 ms)Split 1024 edits, skipped 0 edits. at 1732631477080 (+69 ms)Split 2048 edits, skipped 0 edits. at 1732631477106 (+26 ms)Finishing writing output for hdfs://localhost:44999/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1732631472647/wal.1732631472718 so closing down at 1732631477131 (+25 ms)3 split writer threads finished at 1732631477132 (+1 ms)Closed recovered edits writer path=hdfs://localhost:44999/hbase/data/default/testReplayEditsWrittenIntoWAL/6aa243d9781ecdebfdec5139a938dbfa/recovered.edits/0000000000000000001-wal.1732631472718.temp (wrote 3002 edits, skipped 0 edits in 49 ms) at 1732631477136 (+4 ms)Rename recovered edits hdfs://localhost:44999/hbase/data/default/testReplayEditsWrittenIntoWAL/6aa243d9781ecdebfdec5139a938dbfa/recovered.edits/0000000000000000001-wal.1732631472718.temp to hdfs://localhost:44999/hbase/data/default/testReplayEditsWrittenIntoWAL/6aa243d9781ecdebfdec5139a938dbfa/recovered.edits/0000000000000003002 at 1732631477137 (+1 ms)Processed 3002 edits across 1 Regions in 133 ms; skipped=0; WAL=hdfs://localhost:44999/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1732631472647/wal.1732631472718, size=0, length=0, corrupted=false, cancelled=false at 1732631477137 2024-11-26T14:31:17,140 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:44999/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1732631472647/wal.1732631472718 to hdfs://localhost:44999/hbase/oldWALs/wal.1732631472718 2024-11-26T14:31:17,141 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:44999/hbase/data/default/testReplayEditsWrittenIntoWAL/6aa243d9781ecdebfdec5139a938dbfa/recovered.edits/0000000000000003002 2024-11-26T14:31:17,141 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-26T14:31:17,144 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:44999/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1732631472647, archiveDir=hdfs://localhost:44999/hbase/oldWALs, maxLogs=32 2024-11-26T14:31:17,156 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testreplayeditswrittenintowal-manual,16010,1732631472647/wal.1732631477144, exclude list is [], retry=0 2024-11-26T14:31:17,160 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36307,DS-dd37de78-b3d5-4d7c-8223-4b15101c21bf,DISK] 2024-11-26T14:31:17,160 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38147,DS-44b12d44-38d0-4c76-b56f-3ca8d8caa398,DISK] 2024-11-26T14:31:17,160 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36277,DS-9cb9383d-a6c8-4ba6-83b3-6706331a6539,DISK] 2024-11-26T14:31:17,162 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testreplayeditswrittenintowal-manual,16010,1732631472647/wal.1732631477144 2024-11-26T14:31:17,163 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:34431:34431),(127.0.0.1/127.0.0.1:35081:35081),(127.0.0.1/127.0.0.1:40449:40449)] 2024-11-26T14:31:17,163 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenIntoWAL,,1732631472648.6aa243d9781ecdebfdec5139a938dbfa.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-26T14:31:17,166 INFO [StoreOpener-6aa243d9781ecdebfdec5139a938dbfa-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 6aa243d9781ecdebfdec5139a938dbfa 2024-11-26T14:31:17,168 INFO [StoreOpener-6aa243d9781ecdebfdec5139a938dbfa-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 6aa243d9781ecdebfdec5139a938dbfa columnFamilyName a 2024-11-26T14:31:17,168 DEBUG [StoreOpener-6aa243d9781ecdebfdec5139a938dbfa-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:31:17,169 INFO [StoreOpener-6aa243d9781ecdebfdec5139a938dbfa-1 {}] regionserver.HStore(327): Store=6aa243d9781ecdebfdec5139a938dbfa/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-26T14:31:17,169 INFO [StoreOpener-6aa243d9781ecdebfdec5139a938dbfa-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 6aa243d9781ecdebfdec5139a938dbfa 2024-11-26T14:31:17,171 INFO [StoreOpener-6aa243d9781ecdebfdec5139a938dbfa-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 6aa243d9781ecdebfdec5139a938dbfa columnFamilyName b 2024-11-26T14:31:17,171 DEBUG [StoreOpener-6aa243d9781ecdebfdec5139a938dbfa-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:31:17,172 INFO [StoreOpener-6aa243d9781ecdebfdec5139a938dbfa-1 {}] regionserver.HStore(327): Store=6aa243d9781ecdebfdec5139a938dbfa/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-26T14:31:17,172 INFO [StoreOpener-6aa243d9781ecdebfdec5139a938dbfa-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 6aa243d9781ecdebfdec5139a938dbfa 2024-11-26T14:31:17,173 INFO [StoreOpener-6aa243d9781ecdebfdec5139a938dbfa-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 6aa243d9781ecdebfdec5139a938dbfa columnFamilyName c 2024-11-26T14:31:17,174 DEBUG [StoreOpener-6aa243d9781ecdebfdec5139a938dbfa-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:31:17,174 INFO [StoreOpener-6aa243d9781ecdebfdec5139a938dbfa-1 {}] regionserver.HStore(327): Store=6aa243d9781ecdebfdec5139a938dbfa/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-26T14:31:17,175 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 6aa243d9781ecdebfdec5139a938dbfa 2024-11-26T14:31:17,176 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44999/hbase/data/default/testReplayEditsWrittenIntoWAL/6aa243d9781ecdebfdec5139a938dbfa 2024-11-26T14:31:17,178 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:44999/hbase/data/default/testReplayEditsWrittenIntoWAL/6aa243d9781ecdebfdec5139a938dbfa 2024-11-26T14:31:17,179 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:44999/hbase/data/default/testReplayEditsWrittenIntoWAL/6aa243d9781ecdebfdec5139a938dbfa/recovered.edits/0000000000000003002 2024-11-26T14:31:17,182 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:44999/hbase/data/default/testReplayEditsWrittenIntoWAL/6aa243d9781ecdebfdec5139a938dbfa/recovered.edits/0000000000000003002: isRecoveredEdits=true, hasTagCompression=true, hasValueCompression=false, valueCompressionType=null 2024-11-26T14:31:17,230 INFO [Time-limited test {}] wal.AbstractTestWALReplay$4$1(796): InternalFlushCache Invoked 2024-11-26T14:31:17,595 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 6aa243d9781ecdebfdec5139a938dbfa 3/3 column families, dataSize=42.49 KB heapSize=100.11 KB 2024-11-26T14:31:17,638 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44999/hbase/data/default/testReplayEditsWrittenIntoWAL/6aa243d9781ecdebfdec5139a938dbfa/.tmp/a/dc4a37ef63214e9db018e0505c92fcf1 is 62, key is testReplayEditsWrittenIntoWAL/a:100/1732631472749/Put/seqid=0 2024-11-26T14:31:17,647 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36277 is added to blk_1073741858_1035 (size=50463) 2024-11-26T14:31:17,648 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38147 is added to blk_1073741858_1035 (size=50463) 2024-11-26T14:31:17,648 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36307 is added to blk_1073741858_1035 (size=50463) 2024-11-26T14:31:17,649 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=754 (bloomFilter=true), to=hdfs://localhost:44999/hbase/data/default/testReplayEditsWrittenIntoWAL/6aa243d9781ecdebfdec5139a938dbfa/.tmp/a/dc4a37ef63214e9db018e0505c92fcf1 2024-11-26T14:31:17,658 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44999/hbase/data/default/testReplayEditsWrittenIntoWAL/6aa243d9781ecdebfdec5139a938dbfa/.tmp/a/dc4a37ef63214e9db018e0505c92fcf1 as hdfs://localhost:44999/hbase/data/default/testReplayEditsWrittenIntoWAL/6aa243d9781ecdebfdec5139a938dbfa/a/dc4a37ef63214e9db018e0505c92fcf1 2024-11-26T14:31:17,664 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44999/hbase/data/default/testReplayEditsWrittenIntoWAL/6aa243d9781ecdebfdec5139a938dbfa/a/dc4a37ef63214e9db018e0505c92fcf1, entries=754, sequenceid=754, filesize=49.3 K 2024-11-26T14:31:17,665 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~42.49 KB/43512, heapSize ~99.59 KB/101984, currentSize=0 B/0 for 6aa243d9781ecdebfdec5139a938dbfa in 70ms, sequenceid=754, compaction requested=false; wal=null 2024-11-26T14:31:17,687 INFO [Time-limited test {}] wal.AbstractTestWALReplay$4$1(796): InternalFlushCache Invoked 2024-11-26T14:31:17,687 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 6aa243d9781ecdebfdec5139a938dbfa 3/3 column families, dataSize=42.49 KB heapSize=100.11 KB 2024-11-26T14:31:17,697 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44999/hbase/data/default/testReplayEditsWrittenIntoWAL/6aa243d9781ecdebfdec5139a938dbfa/.tmp/a/5ae8928d4c654656b3db55f59cee2467 is 62, key is testReplayEditsWrittenIntoWAL/a:754/1732631472808/Put/seqid=0 2024-11-26T14:31:17,709 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36307 is added to blk_1073741859_1036 (size=20072) 2024-11-26T14:31:17,709 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38147 is added to blk_1073741859_1036 (size=20072) 2024-11-26T14:31:17,709 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36277 is added to blk_1073741859_1036 (size=20072) 2024-11-26T14:31:18,111 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.93 KB at sequenceid=1508 (bloomFilter=true), to=hdfs://localhost:44999/hbase/data/default/testReplayEditsWrittenIntoWAL/6aa243d9781ecdebfdec5139a938dbfa/.tmp/a/5ae8928d4c654656b3db55f59cee2467 2024-11-26T14:31:18,143 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44999/hbase/data/default/testReplayEditsWrittenIntoWAL/6aa243d9781ecdebfdec5139a938dbfa/.tmp/b/d55cd89e43f64f7289acff5b6f0d16d2 is 62, key is testReplayEditsWrittenIntoWAL/b:100/1732631472836/Put/seqid=0 2024-11-26T14:31:18,150 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36277 is added to blk_1073741860_1037 (size=35835) 2024-11-26T14:31:18,150 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36307 is added to blk_1073741860_1037 (size=35835) 2024-11-26T14:31:18,151 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38147 is added to blk_1073741860_1037 (size=35835) 2024-11-26T14:31:18,151 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=28.56 KB at sequenceid=1508 (bloomFilter=true), to=hdfs://localhost:44999/hbase/data/default/testReplayEditsWrittenIntoWAL/6aa243d9781ecdebfdec5139a938dbfa/.tmp/b/d55cd89e43f64f7289acff5b6f0d16d2 2024-11-26T14:31:18,159 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44999/hbase/data/default/testReplayEditsWrittenIntoWAL/6aa243d9781ecdebfdec5139a938dbfa/.tmp/a/5ae8928d4c654656b3db55f59cee2467 as hdfs://localhost:44999/hbase/data/default/testReplayEditsWrittenIntoWAL/6aa243d9781ecdebfdec5139a938dbfa/a/5ae8928d4c654656b3db55f59cee2467 2024-11-26T14:31:18,166 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44999/hbase/data/default/testReplayEditsWrittenIntoWAL/6aa243d9781ecdebfdec5139a938dbfa/a/5ae8928d4c654656b3db55f59cee2467, entries=246, sequenceid=1508, filesize=19.6 K 2024-11-26T14:31:18,168 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44999/hbase/data/default/testReplayEditsWrittenIntoWAL/6aa243d9781ecdebfdec5139a938dbfa/.tmp/b/d55cd89e43f64f7289acff5b6f0d16d2 as hdfs://localhost:44999/hbase/data/default/testReplayEditsWrittenIntoWAL/6aa243d9781ecdebfdec5139a938dbfa/b/d55cd89e43f64f7289acff5b6f0d16d2 2024-11-26T14:31:18,174 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44999/hbase/data/default/testReplayEditsWrittenIntoWAL/6aa243d9781ecdebfdec5139a938dbfa/b/d55cd89e43f64f7289acff5b6f0d16d2, entries=508, sequenceid=1508, filesize=35.0 K 2024-11-26T14:31:18,175 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~42.49 KB/43512, heapSize ~99.83 KB/102224, currentSize=0 B/0 for 6aa243d9781ecdebfdec5139a938dbfa in 487ms, sequenceid=1508, compaction requested=false; wal=null 2024-11-26T14:31:18,185 INFO [Time-limited test {}] wal.AbstractTestWALReplay$4$1(796): InternalFlushCache Invoked 2024-11-26T14:31:18,185 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 6aa243d9781ecdebfdec5139a938dbfa 3/3 column families, dataSize=42.49 KB heapSize=100.11 KB 2024-11-26T14:31:18,193 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44999/hbase/data/default/testReplayEditsWrittenIntoWAL/6aa243d9781ecdebfdec5139a938dbfa/.tmp/b/6fc9c458f20540019b47b143785ae942 is 62, key is testReplayEditsWrittenIntoWAL/b:508/1732631472864/Put/seqid=0 2024-11-26T14:31:18,200 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36307 is added to blk_1073741861_1038 (size=35082) 2024-11-26T14:31:18,200 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36277 is added to blk_1073741861_1038 (size=35082) 2024-11-26T14:31:18,201 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38147 is added to blk_1073741861_1038 (size=35082) 2024-11-26T14:31:18,201 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=27.87 KB at sequenceid=2262 (bloomFilter=true), to=hdfs://localhost:44999/hbase/data/default/testReplayEditsWrittenIntoWAL/6aa243d9781ecdebfdec5139a938dbfa/.tmp/b/6fc9c458f20540019b47b143785ae942 2024-11-26T14:31:18,224 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44999/hbase/data/default/testReplayEditsWrittenIntoWAL/6aa243d9781ecdebfdec5139a938dbfa/.tmp/c/1e787f6be5bf4820b2c61e47002d4b1a is 62, key is testReplayEditsWrittenIntoWAL/c:100/1732631472907/Put/seqid=0 2024-11-26T14:31:18,232 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36277 is added to blk_1073741862_1039 (size=20825) 2024-11-26T14:31:18,232 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36307 is added to blk_1073741862_1039 (size=20825) 2024-11-26T14:31:18,233 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38147 is added to blk_1073741862_1039 (size=20825) 2024-11-26T14:31:18,233 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=14.63 KB at sequenceid=2262 (bloomFilter=true), to=hdfs://localhost:44999/hbase/data/default/testReplayEditsWrittenIntoWAL/6aa243d9781ecdebfdec5139a938dbfa/.tmp/c/1e787f6be5bf4820b2c61e47002d4b1a 2024-11-26T14:31:18,242 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44999/hbase/data/default/testReplayEditsWrittenIntoWAL/6aa243d9781ecdebfdec5139a938dbfa/.tmp/b/6fc9c458f20540019b47b143785ae942 as hdfs://localhost:44999/hbase/data/default/testReplayEditsWrittenIntoWAL/6aa243d9781ecdebfdec5139a938dbfa/b/6fc9c458f20540019b47b143785ae942 2024-11-26T14:31:18,250 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44999/hbase/data/default/testReplayEditsWrittenIntoWAL/6aa243d9781ecdebfdec5139a938dbfa/b/6fc9c458f20540019b47b143785ae942, entries=492, sequenceid=2262, filesize=34.3 K 2024-11-26T14:31:18,252 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44999/hbase/data/default/testReplayEditsWrittenIntoWAL/6aa243d9781ecdebfdec5139a938dbfa/.tmp/c/1e787f6be5bf4820b2c61e47002d4b1a as hdfs://localhost:44999/hbase/data/default/testReplayEditsWrittenIntoWAL/6aa243d9781ecdebfdec5139a938dbfa/c/1e787f6be5bf4820b2c61e47002d4b1a 2024-11-26T14:31:18,261 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44999/hbase/data/default/testReplayEditsWrittenIntoWAL/6aa243d9781ecdebfdec5139a938dbfa/c/1e787f6be5bf4820b2c61e47002d4b1a, entries=262, sequenceid=2262, filesize=20.3 K 2024-11-26T14:31:18,261 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~42.49 KB/43512, heapSize ~99.83 KB/102224, currentSize=0 B/0 for 6aa243d9781ecdebfdec5139a938dbfa in 76ms, sequenceid=2262, compaction requested=false; wal=null 2024-11-26T14:31:18,273 WARN [Time-limited test {}] regionserver.HRegion(5722): No family for cell testReplayEditsWrittenIntoWAL/another family:testReplayEditsWrittenIntoWAL/1732631472953/Put/vlen=29/seqid=0 in region testReplayEditsWrittenIntoWAL,,1732631472648.6aa243d9781ecdebfdec5139a938dbfa. 2024-11-26T14:31:18,276 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 3001, skipped 1, firstSequenceIdInLog=1, maxSequenceIdInLog=3002, path=hdfs://localhost:44999/hbase/data/default/testReplayEditsWrittenIntoWAL/6aa243d9781ecdebfdec5139a938dbfa/recovered.edits/0000000000000003002 2024-11-26T14:31:18,277 INFO [Time-limited test {}] wal.AbstractTestWALReplay$4$1(796): InternalFlushCache Invoked 2024-11-26T14:31:18,277 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 6aa243d9781ecdebfdec5139a938dbfa 3/3 column families, dataSize=41.85 KB heapSize=98.89 KB 2024-11-26T14:31:18,289 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44999/hbase/data/default/testReplayEditsWrittenIntoWAL/6aa243d9781ecdebfdec5139a938dbfa/.tmp/c/bea25ce60d784282be8b5fd26506541f is 62, key is testReplayEditsWrittenIntoWAL/c:262/1732631472919/Put/seqid=0 2024-11-26T14:31:18,297 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36307 is added to blk_1073741863_1040 (size=50301) 2024-11-26T14:31:18,297 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36277 is added to blk_1073741863_1040 (size=50301) 2024-11-26T14:31:18,297 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38147 is added to blk_1073741863_1040 (size=50301) 2024-11-26T14:31:18,298 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=41.85 KB at sequenceid=3002 (bloomFilter=true), to=hdfs://localhost:44999/hbase/data/default/testReplayEditsWrittenIntoWAL/6aa243d9781ecdebfdec5139a938dbfa/.tmp/c/bea25ce60d784282be8b5fd26506541f 2024-11-26T14:31:18,305 INFO [Time-limited test {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for bea25ce60d784282be8b5fd26506541f 2024-11-26T14:31:18,306 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44999/hbase/data/default/testReplayEditsWrittenIntoWAL/6aa243d9781ecdebfdec5139a938dbfa/.tmp/c/bea25ce60d784282be8b5fd26506541f as hdfs://localhost:44999/hbase/data/default/testReplayEditsWrittenIntoWAL/6aa243d9781ecdebfdec5139a938dbfa/c/bea25ce60d784282be8b5fd26506541f 2024-11-26T14:31:18,312 INFO [Time-limited test {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for bea25ce60d784282be8b5fd26506541f 2024-11-26T14:31:18,312 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44999/hbase/data/default/testReplayEditsWrittenIntoWAL/6aa243d9781ecdebfdec5139a938dbfa/c/bea25ce60d784282be8b5fd26506541f, entries=739, sequenceid=3002, filesize=49.1 K 2024-11-26T14:31:18,313 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~41.85 KB/42854, heapSize ~98.38 KB/100736, currentSize=0 B/0 for 6aa243d9781ecdebfdec5139a938dbfa in 36ms, sequenceid=3002, compaction requested=false; wal=null 2024-11-26T14:31:18,314 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:44999/hbase/data/default/testReplayEditsWrittenIntoWAL/6aa243d9781ecdebfdec5139a938dbfa/recovered.edits/0000000000000003002 2024-11-26T14:31:18,315 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 6aa243d9781ecdebfdec5139a938dbfa 2024-11-26T14:31:18,315 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 6aa243d9781ecdebfdec5139a938dbfa 2024-11-26T14:31:18,316 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsWrittenIntoWAL descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-26T14:31:18,319 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 6aa243d9781ecdebfdec5139a938dbfa 2024-11-26T14:31:18,322 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44999/hbase/data/default/testReplayEditsWrittenIntoWAL/6aa243d9781ecdebfdec5139a938dbfa/recovered.edits/3002.seqid, newMaxSeqId=3002, maxSeqId=1 2024-11-26T14:31:18,324 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 6aa243d9781ecdebfdec5139a938dbfa; next sequenceid=3003; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=204800, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=62889952, jitterRate=-0.0628666877746582}}}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-26T14:31:18,324 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 6aa243d9781ecdebfdec5139a938dbfa: Writing region info on filesystem at 1732631477163Initializing all the Stores at 1732631477164 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732631477164Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732631477166 (+2 ms)Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732631477166Cleaning up temporary data from old regions at 1732631478315 (+1149 ms)Region opened successfully at 1732631478324 (+9 ms) 2024-11-26T14:31:18,391 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 6aa243d9781ecdebfdec5139a938dbfa, disabling compactions & flushes 2024-11-26T14:31:18,391 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsWrittenIntoWAL,,1732631472648.6aa243d9781ecdebfdec5139a938dbfa. 2024-11-26T14:31:18,391 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsWrittenIntoWAL,,1732631472648.6aa243d9781ecdebfdec5139a938dbfa. 2024-11-26T14:31:18,391 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsWrittenIntoWAL,,1732631472648.6aa243d9781ecdebfdec5139a938dbfa. after waiting 0 ms 2024-11-26T14:31:18,391 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsWrittenIntoWAL,,1732631472648.6aa243d9781ecdebfdec5139a938dbfa. 2024-11-26T14:31:18,393 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsWrittenIntoWAL,,1732631472648.6aa243d9781ecdebfdec5139a938dbfa. 2024-11-26T14:31:18,393 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 6aa243d9781ecdebfdec5139a938dbfa: Waiting for close lock at 1732631478391Disabling compacts and flushes for region at 1732631478391Disabling writes for close at 1732631478391Writing region close event to WAL at 1732631478393 (+2 ms)Closed at 1732631478393 2024-11-26T14:31:18,397 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36307 is added to blk_1073741857_1034 (size=93) 2024-11-26T14:31:18,397 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38147 is added to blk_1073741857_1034 (size=93) 2024-11-26T14:31:18,397 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36277 is added to blk_1073741857_1034 (size=93) 2024-11-26T14:31:18,400 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-11-26T14:31:18,401 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL wal:(num 1732631477144) 2024-11-26T14:31:18,416 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestAsyncWALReplayCompressed#testReplayEditsWrittenIntoWAL Thread=402 (was 388) Potentially hanging thread: IPC Client (1987708253) connection to localhost/127.0.0.1:44999 from jenkins.replay.wal.secondtime java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) Potentially hanging thread: LeaseRenewer:jenkins.replay.wal.secondtime@localhost:44999 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Timer for 'HBase' metrics system java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.util.TimerThread.mainLoop(Timer.java:563) java.base@17.0.11/java.util.TimerThread.run(Timer.java:516) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1020740296_22 at /127.0.0.1:39890 [Waiting for operation #6] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-12-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1020740296_22 at /127.0.0.1:47090 [Waiting for operation #8] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/175d8e9e-4f50-9d71-fd80-148275c72391/cluster_49e351ba-434f-d14a-4712-fb0fcff718bc/data/data4 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1020740296_22 at /127.0.0.1:42940 [Waiting for operation #10] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:36561 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704) java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903) app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Client (1987708253) connection to localhost/127.0.0.1:36561 from jenkins java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:37205 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704) java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903) app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-12-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1020740296_22 at /127.0.0.1:42964 [Waiting for operation #26] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/175d8e9e-4f50-9d71-fd80-148275c72391/cluster_49e351ba-434f-d14a-4712-fb0fcff718bc/data/data3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-12-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Client (1987708253) connection to localhost/127.0.0.1:37205 from jenkins java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) - Thread LEAK? -, OpenFileDescriptor=865 (was 785) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=261 (was 224) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=9392 (was 9435) 2024-11-26T14:31:18,430 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestAsyncWALReplayCompressed#test2727 Thread=402, OpenFileDescriptor=865, MaxFileDescriptor=1048576, SystemLoadAverage=261, ProcessCount=11, AvailableMemoryMB=9388 2024-11-26T14:31:18,446 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-26T14:31:18,449 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-26T14:31:18,450 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-26T14:31:18,453 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-77965965, suffix=, logDir=hdfs://localhost:44999/hbase/WALs/hregion-77965965, archiveDir=hdfs://localhost:44999/hbase/oldWALs, maxLogs=32 2024-11-26T14:31:18,466 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-77965965/hregion-77965965.1732631478453, exclude list is [], retry=0 2024-11-26T14:31:18,470 DEBUG [AsyncFSWAL-14-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38147,DS-44b12d44-38d0-4c76-b56f-3ca8d8caa398,DISK] 2024-11-26T14:31:18,470 DEBUG [AsyncFSWAL-14-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36307,DS-dd37de78-b3d5-4d7c-8223-4b15101c21bf,DISK] 2024-11-26T14:31:18,471 DEBUG [AsyncFSWAL-14-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36277,DS-9cb9383d-a6c8-4ba6-83b3-6706331a6539,DISK] 2024-11-26T14:31:18,474 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-77965965/hregion-77965965.1732631478453 2024-11-26T14:31:18,476 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:35081:35081),(127.0.0.1/127.0.0.1:34431:34431),(127.0.0.1/127.0.0.1:40449:40449)] 2024-11-26T14:31:18,476 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => 1889b6eef68f5ac7e5d281da68f46792, NAME => 'test2727,,1732631478447.1889b6eef68f5ac7e5d281da68f46792.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='test2727', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:44999/hbase 2024-11-26T14:31:18,491 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36307 is added to blk_1073741865_1042 (size=43) 2024-11-26T14:31:18,492 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38147 is added to blk_1073741865_1042 (size=43) 2024-11-26T14:31:18,492 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36277 is added to blk_1073741865_1042 (size=43) 2024-11-26T14:31:18,493 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated test2727,,1732631478447.1889b6eef68f5ac7e5d281da68f46792.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-26T14:31:18,495 INFO [StoreOpener-1889b6eef68f5ac7e5d281da68f46792-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 1889b6eef68f5ac7e5d281da68f46792 2024-11-26T14:31:18,497 INFO [StoreOpener-1889b6eef68f5ac7e5d281da68f46792-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1889b6eef68f5ac7e5d281da68f46792 columnFamilyName a 2024-11-26T14:31:18,497 DEBUG [StoreOpener-1889b6eef68f5ac7e5d281da68f46792-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:31:18,498 INFO [StoreOpener-1889b6eef68f5ac7e5d281da68f46792-1 {}] regionserver.HStore(327): Store=1889b6eef68f5ac7e5d281da68f46792/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-26T14:31:18,498 INFO [StoreOpener-1889b6eef68f5ac7e5d281da68f46792-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 1889b6eef68f5ac7e5d281da68f46792 2024-11-26T14:31:18,500 INFO [StoreOpener-1889b6eef68f5ac7e5d281da68f46792-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1889b6eef68f5ac7e5d281da68f46792 columnFamilyName b 2024-11-26T14:31:18,500 DEBUG [StoreOpener-1889b6eef68f5ac7e5d281da68f46792-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:31:18,501 INFO [StoreOpener-1889b6eef68f5ac7e5d281da68f46792-1 {}] regionserver.HStore(327): Store=1889b6eef68f5ac7e5d281da68f46792/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-26T14:31:18,501 INFO [StoreOpener-1889b6eef68f5ac7e5d281da68f46792-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 1889b6eef68f5ac7e5d281da68f46792 2024-11-26T14:31:18,503 INFO [StoreOpener-1889b6eef68f5ac7e5d281da68f46792-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1889b6eef68f5ac7e5d281da68f46792 columnFamilyName c 2024-11-26T14:31:18,503 DEBUG [StoreOpener-1889b6eef68f5ac7e5d281da68f46792-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:31:18,504 INFO [StoreOpener-1889b6eef68f5ac7e5d281da68f46792-1 {}] regionserver.HStore(327): Store=1889b6eef68f5ac7e5d281da68f46792/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-26T14:31:18,504 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 1889b6eef68f5ac7e5d281da68f46792 2024-11-26T14:31:18,505 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44999/hbase/data/default/test2727/1889b6eef68f5ac7e5d281da68f46792 2024-11-26T14:31:18,505 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44999/hbase/data/default/test2727/1889b6eef68f5ac7e5d281da68f46792 2024-11-26T14:31:18,507 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 1889b6eef68f5ac7e5d281da68f46792 2024-11-26T14:31:18,507 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 1889b6eef68f5ac7e5d281da68f46792 2024-11-26T14:31:18,507 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table test2727 descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-26T14:31:18,509 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 1889b6eef68f5ac7e5d281da68f46792 2024-11-26T14:31:18,511 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44999/hbase/data/default/test2727/1889b6eef68f5ac7e5d281da68f46792/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-26T14:31:18,512 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 1889b6eef68f5ac7e5d281da68f46792; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=59941846, jitterRate=-0.10679689049720764}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-26T14:31:18,513 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 1889b6eef68f5ac7e5d281da68f46792: Writing region info on filesystem at 1732631478493Initializing all the Stores at 1732631478495 (+2 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732631478495Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732631478495Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732631478495Cleaning up temporary data from old regions at 1732631478507 (+12 ms)Region opened successfully at 1732631478513 (+6 ms) 2024-11-26T14:31:18,513 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 1889b6eef68f5ac7e5d281da68f46792, disabling compactions & flushes 2024-11-26T14:31:18,513 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region test2727,,1732631478447.1889b6eef68f5ac7e5d281da68f46792. 2024-11-26T14:31:18,513 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on test2727,,1732631478447.1889b6eef68f5ac7e5d281da68f46792. 2024-11-26T14:31:18,513 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on test2727,,1732631478447.1889b6eef68f5ac7e5d281da68f46792. after waiting 0 ms 2024-11-26T14:31:18,513 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region test2727,,1732631478447.1889b6eef68f5ac7e5d281da68f46792. 2024-11-26T14:31:18,514 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed test2727,,1732631478447.1889b6eef68f5ac7e5d281da68f46792. 2024-11-26T14:31:18,514 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 1889b6eef68f5ac7e5d281da68f46792: Waiting for close lock at 1732631478513Disabling compacts and flushes for region at 1732631478513Disabling writes for close at 1732631478513Writing region close event to WAL at 1732631478514 (+1 ms)Closed at 1732631478514 2024-11-26T14:31:18,517 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36307 is added to blk_1073741864_1041 (size=93) 2024-11-26T14:31:18,518 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36277 is added to blk_1073741864_1041 (size=93) 2024-11-26T14:31:18,519 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38147 is added to blk_1073741864_1041 (size=93) 2024-11-26T14:31:18,521 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-11-26T14:31:18,521 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-77965965:(num 1732631478453) 2024-11-26T14:31:18,521 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-26T14:31:18,524 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:44999/hbase/WALs/test2727-manual,16010,1732631478445, archiveDir=hdfs://localhost:44999/hbase/oldWALs, maxLogs=32 2024-11-26T14:31:18,542 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/test2727-manual,16010,1732631478445/wal.1732631478524, exclude list is [], retry=0 2024-11-26T14:31:18,546 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38147,DS-44b12d44-38d0-4c76-b56f-3ca8d8caa398,DISK] 2024-11-26T14:31:18,546 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36277,DS-9cb9383d-a6c8-4ba6-83b3-6706331a6539,DISK] 2024-11-26T14:31:18,547 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36307,DS-dd37de78-b3d5-4d7c-8223-4b15101c21bf,DISK] 2024-11-26T14:31:18,549 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/test2727-manual,16010,1732631478445/wal.1732631478524 2024-11-26T14:31:18,550 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:35081:35081),(127.0.0.1/127.0.0.1:40449:40449),(127.0.0.1/127.0.0.1:34431:34431)] 2024-11-26T14:31:18,686 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38147 is added to blk_1073741866_1043 (size=200357) 2024-11-26T14:31:18,686 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36307 is added to blk_1073741866_1043 (size=200357) 2024-11-26T14:31:18,686 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36277 is added to blk_1073741866_1043 (size=200357) 2024-11-26T14:31:18,704 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:44999/hbase/WALs/test2727-manual,16010,1732631478445/wal.1732631478524, size=195.7 K (200357bytes) 2024-11-26T14:31:18,704 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:44999/hbase/WALs/test2727-manual,16010,1732631478445/wal.1732631478524 2024-11-26T14:31:18,705 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:44999/hbase/WALs/test2727-manual,16010,1732631478445/wal.1732631478524 after 1ms 2024-11-26T14:31:18,708 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:44999/hbase/WALs/test2727-manual,16010,1732631478445/wal.1732631478524: isRecoveredEdits=false, hasTagCompression=true, hasValueCompression=false, valueCompressionType=null 2024-11-26T14:31:18,710 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:44999/hbase/WALs/test2727-manual,16010,1732631478445/wal.1732631478524 took 6ms 2024-11-26T14:31:18,714 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000001-wal.1732631478524.temp 2024-11-26T14:31:18,716 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:44999/hbase/data/default/test2727/1889b6eef68f5ac7e5d281da68f46792/recovered.edits/0000000000000000001-wal.1732631478524.temp 2024-11-26T14:31:18,774 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:44999/hbase/WALs/test2727-manual,16010,1732631478445/wal.1732631478524 so closing down 2024-11-26T14:31:18,774 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-26T14:31:18,774 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-11-26T14:31:18,777 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36277 is added to blk_1073741867_1044 (size=200357) 2024-11-26T14:31:18,777 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38147 is added to blk_1073741867_1044 (size=200357) 2024-11-26T14:31:18,779 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36307 is added to blk_1073741867_1044 (size=200357) 2024-11-26T14:31:18,779 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:44999/hbase/data/default/test2727/1889b6eef68f5ac7e5d281da68f46792/recovered.edits/0000000000000000001-wal.1732631478524.temp (wrote 3000 edits, skipped 0 edits in 23 ms) 2024-11-26T14:31:18,781 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:44999/hbase/data/default/test2727/1889b6eef68f5ac7e5d281da68f46792/recovered.edits/0000000000000000001-wal.1732631478524.temp to hdfs://localhost:44999/hbase/data/default/test2727/1889b6eef68f5ac7e5d281da68f46792/recovered.edits/0000000000000003000 2024-11-26T14:31:18,781 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 3000 edits across 1 Regions in 71 ms; skipped=0; WAL=hdfs://localhost:44999/hbase/WALs/test2727-manual,16010,1732631478445/wal.1732631478524, size=195.7 K, length=200357, corrupted=false, cancelled=false 2024-11-26T14:31:18,781 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:44999/hbase/WALs/test2727-manual,16010,1732631478445/wal.1732631478524, journal: Splitting hdfs://localhost:44999/hbase/WALs/test2727-manual,16010,1732631478445/wal.1732631478524, size=195.7 K (200357bytes) at 1732631478704Creating recovered edits writer path=hdfs://localhost:44999/hbase/data/default/test2727/1889b6eef68f5ac7e5d281da68f46792/recovered.edits/0000000000000000001-wal.1732631478524.temp at 1732631478716 (+12 ms)Split 1024 edits, skipped 0 edits. at 1732631478733 (+17 ms)Split 2048 edits, skipped 0 edits. at 1732631478754 (+21 ms)Finishing writing output for hdfs://localhost:44999/hbase/WALs/test2727-manual,16010,1732631478445/wal.1732631478524 so closing down at 1732631478774 (+20 ms)3 split writer threads finished at 1732631478774Closed recovered edits writer path=hdfs://localhost:44999/hbase/data/default/test2727/1889b6eef68f5ac7e5d281da68f46792/recovered.edits/0000000000000000001-wal.1732631478524.temp (wrote 3000 edits, skipped 0 edits in 23 ms) at 1732631478779 (+5 ms)Rename recovered edits hdfs://localhost:44999/hbase/data/default/test2727/1889b6eef68f5ac7e5d281da68f46792/recovered.edits/0000000000000000001-wal.1732631478524.temp to hdfs://localhost:44999/hbase/data/default/test2727/1889b6eef68f5ac7e5d281da68f46792/recovered.edits/0000000000000003000 at 1732631478781 (+2 ms)Processed 3000 edits across 1 Regions in 71 ms; skipped=0; WAL=hdfs://localhost:44999/hbase/WALs/test2727-manual,16010,1732631478445/wal.1732631478524, size=195.7 K, length=200357, corrupted=false, cancelled=false at 1732631478781 2024-11-26T14:31:18,783 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:44999/hbase/WALs/test2727-manual,16010,1732631478445/wal.1732631478524 to hdfs://localhost:44999/hbase/oldWALs/wal.1732631478524 2024-11-26T14:31:18,785 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:44999/hbase/data/default/test2727/1889b6eef68f5ac7e5d281da68f46792/recovered.edits/0000000000000003000 2024-11-26T14:31:18,785 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-26T14:31:18,787 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:44999/hbase/WALs/test2727-manual,16010,1732631478445, archiveDir=hdfs://localhost:44999/hbase/oldWALs, maxLogs=32 2024-11-26T14:31:18,807 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/test2727-manual,16010,1732631478445/wal.1732631478788, exclude list is [], retry=0 2024-11-26T14:31:18,811 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36277,DS-9cb9383d-a6c8-4ba6-83b3-6706331a6539,DISK] 2024-11-26T14:31:18,811 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38147,DS-44b12d44-38d0-4c76-b56f-3ca8d8caa398,DISK] 2024-11-26T14:31:18,812 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36307,DS-dd37de78-b3d5-4d7c-8223-4b15101c21bf,DISK] 2024-11-26T14:31:18,816 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/test2727-manual,16010,1732631478445/wal.1732631478788 2024-11-26T14:31:18,816 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:40449:40449),(127.0.0.1/127.0.0.1:35081:35081),(127.0.0.1/127.0.0.1:34431:34431)] 2024-11-26T14:31:18,967 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38147 is added to blk_1073741868_1045 (size=200484) 2024-11-26T14:31:18,967 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36307 is added to blk_1073741868_1045 (size=200484) 2024-11-26T14:31:18,969 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36277 is added to blk_1073741868_1045 (size=200484) 2024-11-26T14:31:18,993 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:44999/hbase/WALs/test2727-manual,16010,1732631478445/wal.1732631478788, size=195.8 K (200484bytes) 2024-11-26T14:31:18,993 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:44999/hbase/WALs/test2727-manual,16010,1732631478445/wal.1732631478788 2024-11-26T14:31:18,994 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:44999/hbase/WALs/test2727-manual,16010,1732631478445/wal.1732631478788 after 1ms 2024-11-26T14:31:18,995 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta 2024-11-26T14:31:18,995 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_meta Metrics about Tables on a single HBase RegionServer 2024-11-26T14:31:18,997 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:44999/hbase/WALs/test2727-manual,16010,1732631478445/wal.1732631478788: isRecoveredEdits=false, hasTagCompression=true, hasValueCompression=false, valueCompressionType=null 2024-11-26T14:31:18,999 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-26T14:31:18,999 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint Metrics about HBase RegionObservers 2024-11-26T14:31:18,999 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_master_table_store 2024-11-26T14:31:18,999 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:44999/hbase/WALs/test2727-manual,16010,1732631478445/wal.1732631478788 took 6ms 2024-11-26T14:31:18,999 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_master_table_store Metrics about Tables on a single HBase RegionServer 2024-11-26T14:31:19,001 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testReplayEditsWrittenIntoWAL 2024-11-26T14:31:19,001 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testReplayEditsWrittenIntoWAL Metrics about Tables on a single HBase RegionServer 2024-11-26T14:31:19,014 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000003001-wal.1732631478788.temp 2024-11-26T14:31:19,016 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:44999/hbase/data/default/test2727/1889b6eef68f5ac7e5d281da68f46792/recovered.edits/0000000000000003001-wal.1732631478788.temp 2024-11-26T14:31:19,064 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:44999/hbase/WALs/test2727-manual,16010,1732631478445/wal.1732631478788 so closing down 2024-11-26T14:31:19,065 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-26T14:31:19,065 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-11-26T14:31:19,068 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36277 is added to blk_1073741869_1046 (size=200484) 2024-11-26T14:31:19,068 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36307 is added to blk_1073741869_1046 (size=200484) 2024-11-26T14:31:19,068 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38147 is added to blk_1073741869_1046 (size=200484) 2024-11-26T14:31:19,069 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:44999/hbase/data/default/test2727/1889b6eef68f5ac7e5d281da68f46792/recovered.edits/0000000000000003001-wal.1732631478788.temp (wrote 3000 edits, skipped 0 edits in 19 ms) 2024-11-26T14:31:19,071 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:44999/hbase/data/default/test2727/1889b6eef68f5ac7e5d281da68f46792/recovered.edits/0000000000000003001-wal.1732631478788.temp to hdfs://localhost:44999/hbase/data/default/test2727/1889b6eef68f5ac7e5d281da68f46792/recovered.edits/0000000000000006000 2024-11-26T14:31:19,071 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 3000 edits across 1 Regions in 69 ms; skipped=0; WAL=hdfs://localhost:44999/hbase/WALs/test2727-manual,16010,1732631478445/wal.1732631478788, size=195.8 K, length=200484, corrupted=false, cancelled=false 2024-11-26T14:31:19,071 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:44999/hbase/WALs/test2727-manual,16010,1732631478445/wal.1732631478788, journal: Splitting hdfs://localhost:44999/hbase/WALs/test2727-manual,16010,1732631478445/wal.1732631478788, size=195.8 K (200484bytes) at 1732631478993Creating recovered edits writer path=hdfs://localhost:44999/hbase/data/default/test2727/1889b6eef68f5ac7e5d281da68f46792/recovered.edits/0000000000000003001-wal.1732631478788.temp at 1732631479016 (+23 ms)Split 1024 edits, skipped 0 edits. at 1732631479028 (+12 ms)Split 2048 edits, skipped 0 edits. at 1732631479048 (+20 ms)Finishing writing output for hdfs://localhost:44999/hbase/WALs/test2727-manual,16010,1732631478445/wal.1732631478788 so closing down at 1732631479065 (+17 ms)3 split writer threads finished at 1732631479065Closed recovered edits writer path=hdfs://localhost:44999/hbase/data/default/test2727/1889b6eef68f5ac7e5d281da68f46792/recovered.edits/0000000000000003001-wal.1732631478788.temp (wrote 3000 edits, skipped 0 edits in 19 ms) at 1732631479069 (+4 ms)Rename recovered edits hdfs://localhost:44999/hbase/data/default/test2727/1889b6eef68f5ac7e5d281da68f46792/recovered.edits/0000000000000003001-wal.1732631478788.temp to hdfs://localhost:44999/hbase/data/default/test2727/1889b6eef68f5ac7e5d281da68f46792/recovered.edits/0000000000000006000 at 1732631479071 (+2 ms)Processed 3000 edits across 1 Regions in 69 ms; skipped=0; WAL=hdfs://localhost:44999/hbase/WALs/test2727-manual,16010,1732631478445/wal.1732631478788, size=195.8 K, length=200484, corrupted=false, cancelled=false at 1732631479071 2024-11-26T14:31:19,073 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:44999/hbase/WALs/test2727-manual,16010,1732631478445/wal.1732631478788 to hdfs://localhost:44999/hbase/oldWALs/wal.1732631478788 2024-11-26T14:31:19,074 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:44999/hbase/data/default/test2727/1889b6eef68f5ac7e5d281da68f46792/recovered.edits/0000000000000006000 2024-11-26T14:31:19,075 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-26T14:31:19,077 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:44999/hbase/WALs/test2727-manual,16010,1732631478445, archiveDir=hdfs://localhost:44999/hbase/oldWALs, maxLogs=32 2024-11-26T14:31:19,090 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/test2727-manual,16010,1732631478445/wal.1732631479077, exclude list is [], retry=0 2024-11-26T14:31:19,094 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38147,DS-44b12d44-38d0-4c76-b56f-3ca8d8caa398,DISK] 2024-11-26T14:31:19,094 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36307,DS-dd37de78-b3d5-4d7c-8223-4b15101c21bf,DISK] 2024-11-26T14:31:19,095 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36277,DS-9cb9383d-a6c8-4ba6-83b3-6706331a6539,DISK] 2024-11-26T14:31:19,098 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/test2727-manual,16010,1732631478445/wal.1732631479077 2024-11-26T14:31:19,098 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:35081:35081),(127.0.0.1/127.0.0.1:34431:34431),(127.0.0.1/127.0.0.1:40449:40449)] 2024-11-26T14:31:19,098 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1889b6eef68f5ac7e5d281da68f46792, NAME => 'test2727,,1732631478447.1889b6eef68f5ac7e5d281da68f46792.', STARTKEY => '', ENDKEY => ''} 2024-11-26T14:31:19,099 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated test2727,,1732631478447.1889b6eef68f5ac7e5d281da68f46792.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-26T14:31:19,099 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 1889b6eef68f5ac7e5d281da68f46792 2024-11-26T14:31:19,099 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 1889b6eef68f5ac7e5d281da68f46792 2024-11-26T14:31:19,101 INFO [StoreOpener-1889b6eef68f5ac7e5d281da68f46792-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 1889b6eef68f5ac7e5d281da68f46792 2024-11-26T14:31:19,102 INFO [StoreOpener-1889b6eef68f5ac7e5d281da68f46792-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1889b6eef68f5ac7e5d281da68f46792 columnFamilyName a 2024-11-26T14:31:19,102 DEBUG [StoreOpener-1889b6eef68f5ac7e5d281da68f46792-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:31:19,102 INFO [StoreOpener-1889b6eef68f5ac7e5d281da68f46792-1 {}] regionserver.HStore(327): Store=1889b6eef68f5ac7e5d281da68f46792/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-26T14:31:19,102 INFO [StoreOpener-1889b6eef68f5ac7e5d281da68f46792-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 1889b6eef68f5ac7e5d281da68f46792 2024-11-26T14:31:19,103 INFO [StoreOpener-1889b6eef68f5ac7e5d281da68f46792-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1889b6eef68f5ac7e5d281da68f46792 columnFamilyName b 2024-11-26T14:31:19,103 DEBUG [StoreOpener-1889b6eef68f5ac7e5d281da68f46792-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:31:19,104 INFO [StoreOpener-1889b6eef68f5ac7e5d281da68f46792-1 {}] regionserver.HStore(327): Store=1889b6eef68f5ac7e5d281da68f46792/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-26T14:31:19,104 INFO [StoreOpener-1889b6eef68f5ac7e5d281da68f46792-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 1889b6eef68f5ac7e5d281da68f46792 2024-11-26T14:31:19,105 INFO [StoreOpener-1889b6eef68f5ac7e5d281da68f46792-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1889b6eef68f5ac7e5d281da68f46792 columnFamilyName c 2024-11-26T14:31:19,106 DEBUG [StoreOpener-1889b6eef68f5ac7e5d281da68f46792-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:31:19,106 INFO [StoreOpener-1889b6eef68f5ac7e5d281da68f46792-1 {}] regionserver.HStore(327): Store=1889b6eef68f5ac7e5d281da68f46792/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-26T14:31:19,107 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 1889b6eef68f5ac7e5d281da68f46792 2024-11-26T14:31:19,108 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44999/hbase/data/default/test2727/1889b6eef68f5ac7e5d281da68f46792 2024-11-26T14:31:19,110 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 2 recovered edits file(s) under hdfs://localhost:44999/hbase/data/default/test2727/1889b6eef68f5ac7e5d281da68f46792 2024-11-26T14:31:19,111 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:44999/hbase/data/default/test2727/1889b6eef68f5ac7e5d281da68f46792/recovered.edits/0000000000000003000 2024-11-26T14:31:19,114 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:44999/hbase/data/default/test2727/1889b6eef68f5ac7e5d281da68f46792/recovered.edits/0000000000000003000: isRecoveredEdits=true, hasTagCompression=true, hasValueCompression=false, valueCompressionType=null 2024-11-26T14:31:19,165 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 3000, skipped 0, firstSequenceIdInLog=1, maxSequenceIdInLog=3000, path=hdfs://localhost:44999/hbase/data/default/test2727/1889b6eef68f5ac7e5d281da68f46792/recovered.edits/0000000000000003000 2024-11-26T14:31:19,166 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:44999/hbase/data/default/test2727/1889b6eef68f5ac7e5d281da68f46792/recovered.edits/0000000000000006000 2024-11-26T14:31:19,169 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:44999/hbase/data/default/test2727/1889b6eef68f5ac7e5d281da68f46792/recovered.edits/0000000000000006000: isRecoveredEdits=true, hasTagCompression=true, hasValueCompression=false, valueCompressionType=null 2024-11-26T14:31:19,227 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 3000, skipped 0, firstSequenceIdInLog=3001, maxSequenceIdInLog=6000, path=hdfs://localhost:44999/hbase/data/default/test2727/1889b6eef68f5ac7e5d281da68f46792/recovered.edits/0000000000000006000 2024-11-26T14:31:19,227 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 1889b6eef68f5ac7e5d281da68f46792 3/3 column families, dataSize=215.51 KB heapSize=657 KB 2024-11-26T14:31:19,252 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44999/hbase/data/default/test2727/1889b6eef68f5ac7e5d281da68f46792/.tmp/a/3c98688122e946b3846c336d83955c1c is 41, key is test2727/a:100/1732631478823/Put/seqid=0 2024-11-26T14:31:19,258 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36307 is added to blk_1073741871_1048 (size=84227) 2024-11-26T14:31:19,258 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38147 is added to blk_1073741871_1048 (size=84227) 2024-11-26T14:31:19,259 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36277 is added to blk_1073741871_1048 (size=84227) 2024-11-26T14:31:19,259 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=71.84 KB at sequenceid=6000 (bloomFilter=true), to=hdfs://localhost:44999/hbase/data/default/test2727/1889b6eef68f5ac7e5d281da68f46792/.tmp/a/3c98688122e946b3846c336d83955c1c 2024-11-26T14:31:19,297 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44999/hbase/data/default/test2727/1889b6eef68f5ac7e5d281da68f46792/.tmp/b/edb799c5c75f4bef962191dc41647983 is 41, key is test2727/b:100/1732631478878/Put/seqid=0 2024-11-26T14:31:19,300 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36277 is added to blk_1073741872_1049 (size=84609) 2024-11-26T14:31:19,300 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38147 is added to blk_1073741872_1049 (size=84609) 2024-11-26T14:31:19,301 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36307 is added to blk_1073741872_1049 (size=84609) 2024-11-26T14:31:19,302 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=71.84 KB at sequenceid=6000 (bloomFilter=true), to=hdfs://localhost:44999/hbase/data/default/test2727/1889b6eef68f5ac7e5d281da68f46792/.tmp/b/edb799c5c75f4bef962191dc41647983 2024-11-26T14:31:19,329 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44999/hbase/data/default/test2727/1889b6eef68f5ac7e5d281da68f46792/.tmp/c/46b6f7ee026344fd980d557244779459 is 41, key is test2727/c:100/1732631478920/Put/seqid=0 2024-11-26T14:31:19,335 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36277 is added to blk_1073741873_1050 (size=84609) 2024-11-26T14:31:19,335 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36307 is added to blk_1073741873_1050 (size=84609) 2024-11-26T14:31:19,336 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38147 is added to blk_1073741873_1050 (size=84609) 2024-11-26T14:31:19,336 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=71.84 KB at sequenceid=6000 (bloomFilter=true), to=hdfs://localhost:44999/hbase/data/default/test2727/1889b6eef68f5ac7e5d281da68f46792/.tmp/c/46b6f7ee026344fd980d557244779459 2024-11-26T14:31:19,343 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44999/hbase/data/default/test2727/1889b6eef68f5ac7e5d281da68f46792/.tmp/a/3c98688122e946b3846c336d83955c1c as hdfs://localhost:44999/hbase/data/default/test2727/1889b6eef68f5ac7e5d281da68f46792/a/3c98688122e946b3846c336d83955c1c 2024-11-26T14:31:19,352 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44999/hbase/data/default/test2727/1889b6eef68f5ac7e5d281da68f46792/a/3c98688122e946b3846c336d83955c1c, entries=2000, sequenceid=6000, filesize=82.3 K 2024-11-26T14:31:19,353 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44999/hbase/data/default/test2727/1889b6eef68f5ac7e5d281da68f46792/.tmp/b/edb799c5c75f4bef962191dc41647983 as hdfs://localhost:44999/hbase/data/default/test2727/1889b6eef68f5ac7e5d281da68f46792/b/edb799c5c75f4bef962191dc41647983 2024-11-26T14:31:19,361 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44999/hbase/data/default/test2727/1889b6eef68f5ac7e5d281da68f46792/b/edb799c5c75f4bef962191dc41647983, entries=2000, sequenceid=6000, filesize=82.6 K 2024-11-26T14:31:19,363 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44999/hbase/data/default/test2727/1889b6eef68f5ac7e5d281da68f46792/.tmp/c/46b6f7ee026344fd980d557244779459 as hdfs://localhost:44999/hbase/data/default/test2727/1889b6eef68f5ac7e5d281da68f46792/c/46b6f7ee026344fd980d557244779459 2024-11-26T14:31:19,371 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44999/hbase/data/default/test2727/1889b6eef68f5ac7e5d281da68f46792/c/46b6f7ee026344fd980d557244779459, entries=2000, sequenceid=6000, filesize=82.6 K 2024-11-26T14:31:19,372 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~215.51 KB/220680, heapSize ~656.95 KB/672720, currentSize=0 B/0 for 1889b6eef68f5ac7e5d281da68f46792 in 144ms, sequenceid=6000, compaction requested=false; wal=null 2024-11-26T14:31:19,373 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:44999/hbase/data/default/test2727/1889b6eef68f5ac7e5d281da68f46792/recovered.edits/0000000000000003000 2024-11-26T14:31:19,373 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:44999/hbase/data/default/test2727/1889b6eef68f5ac7e5d281da68f46792/recovered.edits/0000000000000006000 2024-11-26T14:31:19,375 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 1889b6eef68f5ac7e5d281da68f46792 2024-11-26T14:31:19,375 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 1889b6eef68f5ac7e5d281da68f46792 2024-11-26T14:31:19,376 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table test2727 descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-26T14:31:19,378 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 1889b6eef68f5ac7e5d281da68f46792 2024-11-26T14:31:19,381 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44999/hbase/data/default/test2727/1889b6eef68f5ac7e5d281da68f46792/recovered.edits/6000.seqid, newMaxSeqId=6000, maxSeqId=1 2024-11-26T14:31:19,382 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 1889b6eef68f5ac7e5d281da68f46792; next sequenceid=6001; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=74857934, jitterRate=0.11547014117240906}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-26T14:31:19,383 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 1889b6eef68f5ac7e5d281da68f46792: Writing region info on filesystem at 1732631479099Initializing all the Stores at 1732631479100 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732631479100Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732631479101 (+1 ms)Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732631479101Obtaining lock to block concurrent updates at 1732631479227 (+126 ms)Preparing flush snapshotting stores in 1889b6eef68f5ac7e5d281da68f46792 at 1732631479227Finished memstore snapshotting test2727,,1732631478447.1889b6eef68f5ac7e5d281da68f46792., syncing WAL and waiting on mvcc, flushsize=dataSize=220680, getHeapSize=672720, getOffHeapSize=0, getCellsCount=6000 at 1732631479228 (+1 ms)Flushing stores of test2727,,1732631478447.1889b6eef68f5ac7e5d281da68f46792. at 1732631479228Flushing 1889b6eef68f5ac7e5d281da68f46792/a: creating writer at 1732631479228Flushing 1889b6eef68f5ac7e5d281da68f46792/a: appending metadata at 1732631479251 (+23 ms)Flushing 1889b6eef68f5ac7e5d281da68f46792/a: closing flushed file at 1732631479251Flushing 1889b6eef68f5ac7e5d281da68f46792/b: creating writer at 1732631479267 (+16 ms)Flushing 1889b6eef68f5ac7e5d281da68f46792/b: appending metadata at 1732631479294 (+27 ms)Flushing 1889b6eef68f5ac7e5d281da68f46792/b: closing flushed file at 1732631479294Flushing 1889b6eef68f5ac7e5d281da68f46792/c: creating writer at 1732631479308 (+14 ms)Flushing 1889b6eef68f5ac7e5d281da68f46792/c: appending metadata at 1732631479329 (+21 ms)Flushing 1889b6eef68f5ac7e5d281da68f46792/c: closing flushed file at 1732631479329Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@32a218a5: reopening flushed file at 1732631479342 (+13 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@4bf9fb08: reopening flushed file at 1732631479352 (+10 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@6073c67c: reopening flushed file at 1732631479361 (+9 ms)Finished flush of dataSize ~215.51 KB/220680, heapSize ~656.95 KB/672720, currentSize=0 B/0 for 1889b6eef68f5ac7e5d281da68f46792 in 144ms, sequenceid=6000, compaction requested=false; wal=null at 1732631479372 (+11 ms)Cleaning up temporary data from old regions at 1732631479375 (+3 ms)Region opened successfully at 1732631479383 (+8 ms) 2024-11-26T14:31:19,385 DEBUG [Time-limited test {}] wal.AbstractTestWALReplay(320): region.getOpenSeqNum(): 6001, wal3.id: 0 2024-11-26T14:31:19,385 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 1889b6eef68f5ac7e5d281da68f46792, disabling compactions & flushes 2024-11-26T14:31:19,385 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region test2727,,1732631478447.1889b6eef68f5ac7e5d281da68f46792. 2024-11-26T14:31:19,385 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on test2727,,1732631478447.1889b6eef68f5ac7e5d281da68f46792. 2024-11-26T14:31:19,385 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on test2727,,1732631478447.1889b6eef68f5ac7e5d281da68f46792. after waiting 0 ms 2024-11-26T14:31:19,385 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region test2727,,1732631478447.1889b6eef68f5ac7e5d281da68f46792. 2024-11-26T14:31:19,387 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed test2727,,1732631478447.1889b6eef68f5ac7e5d281da68f46792. 2024-11-26T14:31:19,387 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 1889b6eef68f5ac7e5d281da68f46792: Waiting for close lock at 1732631479385Disabling compacts and flushes for region at 1732631479385Disabling writes for close at 1732631479385Writing region close event to WAL at 1732631479387 (+2 ms)Closed at 1732631479387 2024-11-26T14:31:19,390 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36307 is added to blk_1073741870_1047 (size=93) 2024-11-26T14:31:19,390 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38147 is added to blk_1073741870_1047 (size=93) 2024-11-26T14:31:19,391 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36277 is added to blk_1073741870_1047 (size=93) 2024-11-26T14:31:19,393 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-11-26T14:31:19,393 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL wal:(num 1732631479077) 2024-11-26T14:31:19,405 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestAsyncWALReplayCompressed#test2727 Thread=404 (was 402) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_2043263076_22 at /127.0.0.1:47090 [Waiting for operation #10] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_2043263076_22 at /127.0.0.1:43058 [Waiting for operation #11] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-14-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-14-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_2043263076_22 at /127.0.0.1:39988 [Waiting for operation #5] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-14-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=927 (was 865) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=261 (was 261), ProcessCount=11 (was 11), AvailableMemoryMB=9174 (was 9388) 2024-11-26T14:31:19,416 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestAsyncWALReplayCompressed#testSequentialEditLogSeqNum Thread=404, OpenFileDescriptor=927, MaxFileDescriptor=1048576, SystemLoadAverage=261, ProcessCount=11, AvailableMemoryMB=9173 2024-11-26T14:31:19,432 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-26T14:31:19,440 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:44999/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1732631479431, archiveDir=hdfs://localhost:44999/hbase/oldWALs, maxLogs=32 2024-11-26T14:31:19,441 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732631479440 2024-11-26T14:31:19,448 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testsequentialeditlogseqnum-manual,16010,1732631479431/wal.1732631479440 2024-11-26T14:31:19,456 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new MockWAL writer with pipeline: [(127.0.0.1/127.0.0.1:35081:35081),(127.0.0.1/127.0.0.1:40449:40449),(127.0.0.1/127.0.0.1:34431:34431)] 2024-11-26T14:31:19,457 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => b46dd0b265bbd02ab81780412e8186b7, NAME => 'testSequentialEditLogSeqNum,,1732631479433.b46dd0b265bbd02ab81780412e8186b7.', STARTKEY => '', ENDKEY => ''} 2024-11-26T14:31:19,457 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testSequentialEditLogSeqNum,,1732631479433.b46dd0b265bbd02ab81780412e8186b7.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-26T14:31:19,457 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for b46dd0b265bbd02ab81780412e8186b7 2024-11-26T14:31:19,458 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for b46dd0b265bbd02ab81780412e8186b7 2024-11-26T14:31:19,458 WARN [Time-limited test {}] regionserver.HRegionFileSystem(836): hdfs://localhost:44999/hbase/data/default/testSequentialEditLogSeqNum/b46dd0b265bbd02ab81780412e8186b7 doesn't exist for region: b46dd0b265bbd02ab81780412e8186b7 on table testSequentialEditLogSeqNum 2024-11-26T14:31:19,459 WARN [Time-limited test {}] regionserver.HRegionFileSystem(854): .regioninfo file not found for region: b46dd0b265bbd02ab81780412e8186b7 on table testSequentialEditLogSeqNum 2024-11-26T14:31:19,469 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36307 is added to blk_1073741875_1052 (size=62) 2024-11-26T14:31:19,470 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38147 is added to blk_1073741875_1052 (size=62) 2024-11-26T14:31:19,471 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36277 is added to blk_1073741875_1052 (size=62) 2024-11-26T14:31:19,473 INFO [StoreOpener-b46dd0b265bbd02ab81780412e8186b7-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region b46dd0b265bbd02ab81780412e8186b7 2024-11-26T14:31:19,474 INFO [StoreOpener-b46dd0b265bbd02ab81780412e8186b7-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region b46dd0b265bbd02ab81780412e8186b7 columnFamilyName a 2024-11-26T14:31:19,474 DEBUG [StoreOpener-b46dd0b265bbd02ab81780412e8186b7-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:31:19,475 INFO [StoreOpener-b46dd0b265bbd02ab81780412e8186b7-1 {}] regionserver.HStore(327): Store=b46dd0b265bbd02ab81780412e8186b7/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-26T14:31:19,475 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for b46dd0b265bbd02ab81780412e8186b7 2024-11-26T14:31:19,476 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44999/hbase/data/default/testSequentialEditLogSeqNum/b46dd0b265bbd02ab81780412e8186b7 2024-11-26T14:31:19,476 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44999/hbase/data/default/testSequentialEditLogSeqNum/b46dd0b265bbd02ab81780412e8186b7 2024-11-26T14:31:19,477 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for b46dd0b265bbd02ab81780412e8186b7 2024-11-26T14:31:19,477 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for b46dd0b265bbd02ab81780412e8186b7 2024-11-26T14:31:19,479 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for b46dd0b265bbd02ab81780412e8186b7 2024-11-26T14:31:19,482 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44999/hbase/data/default/testSequentialEditLogSeqNum/b46dd0b265bbd02ab81780412e8186b7/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-26T14:31:19,482 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened b46dd0b265bbd02ab81780412e8186b7; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=70279262, jitterRate=0.04724261164665222}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-26T14:31:19,484 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for b46dd0b265bbd02ab81780412e8186b7: Writing region info on filesystem at 1732631479458Initializing all the Stores at 1732631479472 (+14 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732631479472Cleaning up temporary data from old regions at 1732631479477 (+5 ms)Region opened successfully at 1732631479484 (+7 ms) 2024-11-26T14:31:19,498 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing b46dd0b265bbd02ab81780412e8186b7 1/1 column families, dataSize=770 B heapSize=1.73 KB 2024-11-26T14:31:19,519 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44999/hbase/data/default/testSequentialEditLogSeqNum/b46dd0b265bbd02ab81780412e8186b7/.tmp/a/35fd850af25a4087b6d04a2d367ce1cc is 81, key is testSequentialEditLogSeqNum/a:x0/1732631479484/Put/seqid=0 2024-11-26T14:31:19,526 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38147 is added to blk_1073741876_1053 (size=5833) 2024-11-26T14:31:19,526 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36307 is added to blk_1073741876_1053 (size=5833) 2024-11-26T14:31:19,527 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36277 is added to blk_1073741876_1053 (size=5833) 2024-11-26T14:31:19,527 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=770 B at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:44999/hbase/data/default/testSequentialEditLogSeqNum/b46dd0b265bbd02ab81780412e8186b7/.tmp/a/35fd850af25a4087b6d04a2d367ce1cc 2024-11-26T14:31:19,534 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44999/hbase/data/default/testSequentialEditLogSeqNum/b46dd0b265bbd02ab81780412e8186b7/.tmp/a/35fd850af25a4087b6d04a2d367ce1cc as hdfs://localhost:44999/hbase/data/default/testSequentialEditLogSeqNum/b46dd0b265bbd02ab81780412e8186b7/a/35fd850af25a4087b6d04a2d367ce1cc 2024-11-26T14:31:19,541 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44999/hbase/data/default/testSequentialEditLogSeqNum/b46dd0b265bbd02ab81780412e8186b7/a/35fd850af25a4087b6d04a2d367ce1cc, entries=10, sequenceid=13, filesize=5.7 K 2024-11-26T14:31:19,542 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~770 B/770, heapSize ~1.72 KB/1760, currentSize=0 B/0 for b46dd0b265bbd02ab81780412e8186b7 in 45ms, sequenceid=13, compaction requested=false 2024-11-26T14:31:19,542 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for b46dd0b265bbd02ab81780412e8186b7: 2024-11-26T14:31:19,551 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T14:31:19,551 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T14:31:19,551 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T14:31:19,551 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T14:31:19,551 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T14:31:19,554 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36307 is added to blk_1073741874_1051 (size=1616) 2024-11-26T14:31:19,554 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36277 is added to blk_1073741874_1051 (size=1616) 2024-11-26T14:31:19,555 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38147 is added to blk_1073741874_1051 (size=1616) 2024-11-26T14:31:19,578 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:44999/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1732631479431/wal.1732631479440, size=1.6 K (1616bytes) 2024-11-26T14:31:19,578 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:44999/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1732631479431/wal.1732631479440 2024-11-26T14:31:19,578 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:44999/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1732631479431/wal.1732631479440 after 0ms 2024-11-26T14:31:19,582 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:44999/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1732631479431/wal.1732631479440: isRecoveredEdits=false, hasTagCompression=true, hasValueCompression=false, valueCompressionType=null 2024-11-26T14:31:19,582 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:44999/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1732631479431/wal.1732631479440 took 5ms 2024-11-26T14:31:19,585 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:44999/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1732631479431/wal.1732631479440 so closing down 2024-11-26T14:31:19,585 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-26T14:31:19,586 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000003-wal.1732631479440.temp 2024-11-26T14:31:19,588 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:44999/hbase/data/default/testSequentialEditLogSeqNum/b46dd0b265bbd02ab81780412e8186b7/recovered.edits/0000000000000000003-wal.1732631479440.temp 2024-11-26T14:31:19,588 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-11-26T14:31:19,600 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36307 is added to blk_1073741877_1054 (size=1175) 2024-11-26T14:31:19,601 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38147 is added to blk_1073741877_1054 (size=1175) 2024-11-26T14:31:19,601 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36277 is added to blk_1073741877_1054 (size=1175) 2024-11-26T14:31:19,602 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:44999/hbase/data/default/testSequentialEditLogSeqNum/b46dd0b265bbd02ab81780412e8186b7/recovered.edits/0000000000000000003-wal.1732631479440.temp (wrote 15 edits, skipped 0 edits in 0 ms) 2024-11-26T14:31:19,603 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:44999/hbase/data/default/testSequentialEditLogSeqNum/b46dd0b265bbd02ab81780412e8186b7/recovered.edits/0000000000000000003-wal.1732631479440.temp to hdfs://localhost:44999/hbase/data/default/testSequentialEditLogSeqNum/b46dd0b265bbd02ab81780412e8186b7/recovered.edits/0000000000000000020 2024-11-26T14:31:19,603 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 17 edits across 1 Regions in 20 ms; skipped=2; WAL=hdfs://localhost:44999/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1732631479431/wal.1732631479440, size=1.6 K, length=1616, corrupted=false, cancelled=false 2024-11-26T14:31:19,603 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:44999/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1732631479431/wal.1732631479440, journal: Splitting hdfs://localhost:44999/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1732631479431/wal.1732631479440, size=1.6 K (1616bytes) at 1732631479578Finishing writing output for hdfs://localhost:44999/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1732631479431/wal.1732631479440 so closing down at 1732631479585 (+7 ms)Creating recovered edits writer path=hdfs://localhost:44999/hbase/data/default/testSequentialEditLogSeqNum/b46dd0b265bbd02ab81780412e8186b7/recovered.edits/0000000000000000003-wal.1732631479440.temp at 1732631479588 (+3 ms)3 split writer threads finished at 1732631479588Closed recovered edits writer path=hdfs://localhost:44999/hbase/data/default/testSequentialEditLogSeqNum/b46dd0b265bbd02ab81780412e8186b7/recovered.edits/0000000000000000003-wal.1732631479440.temp (wrote 15 edits, skipped 0 edits in 0 ms) at 1732631479602 (+14 ms)Rename recovered edits hdfs://localhost:44999/hbase/data/default/testSequentialEditLogSeqNum/b46dd0b265bbd02ab81780412e8186b7/recovered.edits/0000000000000000003-wal.1732631479440.temp to hdfs://localhost:44999/hbase/data/default/testSequentialEditLogSeqNum/b46dd0b265bbd02ab81780412e8186b7/recovered.edits/0000000000000000020 at 1732631479603 (+1 ms)Processed 17 edits across 1 Regions in 20 ms; skipped=2; WAL=hdfs://localhost:44999/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1732631479431/wal.1732631479440, size=1.6 K, length=1616, corrupted=false, cancelled=false at 1732631479603 2024-11-26T14:31:19,619 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestAsyncWALReplayCompressed#testSequentialEditLogSeqNum Thread=409 (was 404) Potentially hanging thread: sync.2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441) Potentially hanging thread: sync.1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_2043263076_22 at /127.0.0.1:47090 [Waiting for operation #12] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: sync.4 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441) Potentially hanging thread: sync.0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_2043263076_22 at /127.0.0.1:43058 [Waiting for operation #15] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: sync.3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441) - Thread LEAK? -, OpenFileDescriptor=963 (was 927) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=261 (was 261), ProcessCount=11 (was 11), AvailableMemoryMB=9153 (was 9173) 2024-11-26T14:31:19,631 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestAsyncWALReplayCompressed#testRegionMadeOfBulkLoadedFilesOnly Thread=409, OpenFileDescriptor=963, MaxFileDescriptor=1048576, SystemLoadAverage=261, ProcessCount=11, AvailableMemoryMB=9152 2024-11-26T14:31:19,647 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-26T14:31:19,650 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-26T14:31:19,669 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-26T14:31:19,672 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-74097924, suffix=, logDir=hdfs://localhost:44999/hbase/WALs/hregion-74097924, archiveDir=hdfs://localhost:44999/hbase/oldWALs, maxLogs=32 2024-11-26T14:31:19,685 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-74097924/hregion-74097924.1732631479672, exclude list is [], retry=0 2024-11-26T14:31:19,689 DEBUG [AsyncFSWAL-17-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36307,DS-dd37de78-b3d5-4d7c-8223-4b15101c21bf,DISK] 2024-11-26T14:31:19,689 DEBUG [AsyncFSWAL-17-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38147,DS-44b12d44-38d0-4c76-b56f-3ca8d8caa398,DISK] 2024-11-26T14:31:19,690 DEBUG [AsyncFSWAL-17-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36277,DS-9cb9383d-a6c8-4ba6-83b3-6706331a6539,DISK] 2024-11-26T14:31:19,693 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-74097924/hregion-74097924.1732631479672 2024-11-26T14:31:19,694 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:34431:34431),(127.0.0.1/127.0.0.1:35081:35081),(127.0.0.1/127.0.0.1:40449:40449)] 2024-11-26T14:31:19,694 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => 339879ea4311256af1be3b960184b7f5, NAME => 'testRegionMadeOfBulkLoadedFilesOnly,,1732631479648.339879ea4311256af1be3b960184b7f5.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testRegionMadeOfBulkLoadedFilesOnly', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:44999/hbase 2024-11-26T14:31:19,709 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38147 is added to blk_1073741879_1056 (size=70) 2024-11-26T14:31:19,709 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36307 is added to blk_1073741879_1056 (size=70) 2024-11-26T14:31:19,709 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36277 is added to blk_1073741879_1056 (size=70) 2024-11-26T14:31:19,710 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testRegionMadeOfBulkLoadedFilesOnly,,1732631479648.339879ea4311256af1be3b960184b7f5.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-26T14:31:19,738 INFO [StoreOpener-339879ea4311256af1be3b960184b7f5-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 339879ea4311256af1be3b960184b7f5 2024-11-26T14:31:19,748 INFO [StoreOpener-339879ea4311256af1be3b960184b7f5-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 339879ea4311256af1be3b960184b7f5 columnFamilyName a 2024-11-26T14:31:19,748 DEBUG [StoreOpener-339879ea4311256af1be3b960184b7f5-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:31:19,749 INFO [StoreOpener-339879ea4311256af1be3b960184b7f5-1 {}] regionserver.HStore(327): Store=339879ea4311256af1be3b960184b7f5/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-26T14:31:19,749 INFO [StoreOpener-339879ea4311256af1be3b960184b7f5-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 339879ea4311256af1be3b960184b7f5 2024-11-26T14:31:19,751 INFO [StoreOpener-339879ea4311256af1be3b960184b7f5-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 339879ea4311256af1be3b960184b7f5 columnFamilyName b 2024-11-26T14:31:19,751 DEBUG [StoreOpener-339879ea4311256af1be3b960184b7f5-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:31:19,752 INFO [StoreOpener-339879ea4311256af1be3b960184b7f5-1 {}] regionserver.HStore(327): Store=339879ea4311256af1be3b960184b7f5/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-26T14:31:19,752 INFO [StoreOpener-339879ea4311256af1be3b960184b7f5-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 339879ea4311256af1be3b960184b7f5 2024-11-26T14:31:19,754 INFO [StoreOpener-339879ea4311256af1be3b960184b7f5-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 339879ea4311256af1be3b960184b7f5 columnFamilyName c 2024-11-26T14:31:19,754 DEBUG [StoreOpener-339879ea4311256af1be3b960184b7f5-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:31:19,755 INFO [StoreOpener-339879ea4311256af1be3b960184b7f5-1 {}] regionserver.HStore(327): Store=339879ea4311256af1be3b960184b7f5/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-26T14:31:19,755 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 339879ea4311256af1be3b960184b7f5 2024-11-26T14:31:19,756 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44999/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/339879ea4311256af1be3b960184b7f5 2024-11-26T14:31:19,757 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44999/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/339879ea4311256af1be3b960184b7f5 2024-11-26T14:31:19,758 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 339879ea4311256af1be3b960184b7f5 2024-11-26T14:31:19,758 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 339879ea4311256af1be3b960184b7f5 2024-11-26T14:31:19,759 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testRegionMadeOfBulkLoadedFilesOnly descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-26T14:31:19,760 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 339879ea4311256af1be3b960184b7f5 2024-11-26T14:31:19,763 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44999/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/339879ea4311256af1be3b960184b7f5/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-26T14:31:19,764 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 339879ea4311256af1be3b960184b7f5; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=65773741, jitterRate=-0.019894883036613464}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-26T14:31:19,765 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 339879ea4311256af1be3b960184b7f5: Writing region info on filesystem at 1732631479710Initializing all the Stores at 1732631479711 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732631479711Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732631479711Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732631479738 (+27 ms)Cleaning up temporary data from old regions at 1732631479758 (+20 ms)Region opened successfully at 1732631479765 (+7 ms) 2024-11-26T14:31:19,765 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 339879ea4311256af1be3b960184b7f5, disabling compactions & flushes 2024-11-26T14:31:19,766 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testRegionMadeOfBulkLoadedFilesOnly,,1732631479648.339879ea4311256af1be3b960184b7f5. 2024-11-26T14:31:19,766 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testRegionMadeOfBulkLoadedFilesOnly,,1732631479648.339879ea4311256af1be3b960184b7f5. 2024-11-26T14:31:19,766 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testRegionMadeOfBulkLoadedFilesOnly,,1732631479648.339879ea4311256af1be3b960184b7f5. after waiting 0 ms 2024-11-26T14:31:19,766 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testRegionMadeOfBulkLoadedFilesOnly,,1732631479648.339879ea4311256af1be3b960184b7f5. 2024-11-26T14:31:19,766 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testRegionMadeOfBulkLoadedFilesOnly,,1732631479648.339879ea4311256af1be3b960184b7f5. 2024-11-26T14:31:19,766 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 339879ea4311256af1be3b960184b7f5: Waiting for close lock at 1732631479765Disabling compacts and flushes for region at 1732631479765Disabling writes for close at 1732631479766 (+1 ms)Writing region close event to WAL at 1732631479766Closed at 1732631479766 2024-11-26T14:31:19,772 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36277 is added to blk_1073741878_1055 (size=93) 2024-11-26T14:31:19,772 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38147 is added to blk_1073741878_1055 (size=93) 2024-11-26T14:31:19,772 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36307 is added to blk_1073741878_1055 (size=93) 2024-11-26T14:31:19,775 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-11-26T14:31:19,775 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-74097924:(num 1732631479672) 2024-11-26T14:31:19,776 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-26T14:31:19,778 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:44999/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1732631479646, archiveDir=hdfs://localhost:44999/hbase/oldWALs, maxLogs=32 2024-11-26T14:31:19,798 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1732631479646/wal.1732631479779, exclude list is [], retry=0 2024-11-26T14:31:19,803 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36307,DS-dd37de78-b3d5-4d7c-8223-4b15101c21bf,DISK] 2024-11-26T14:31:19,803 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36277,DS-9cb9383d-a6c8-4ba6-83b3-6706331a6539,DISK] 2024-11-26T14:31:19,803 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38147,DS-44b12d44-38d0-4c76-b56f-3ca8d8caa398,DISK] 2024-11-26T14:31:19,807 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1732631479646/wal.1732631479779 2024-11-26T14:31:19,807 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:34431:34431),(127.0.0.1/127.0.0.1:40449:40449),(127.0.0.1/127.0.0.1:35081:35081)] 2024-11-26T14:31:19,807 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 339879ea4311256af1be3b960184b7f5, NAME => 'testRegionMadeOfBulkLoadedFilesOnly,,1732631479648.339879ea4311256af1be3b960184b7f5.', STARTKEY => '', ENDKEY => ''} 2024-11-26T14:31:19,808 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testRegionMadeOfBulkLoadedFilesOnly,,1732631479648.339879ea4311256af1be3b960184b7f5.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-26T14:31:19,808 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 339879ea4311256af1be3b960184b7f5 2024-11-26T14:31:19,808 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 339879ea4311256af1be3b960184b7f5 2024-11-26T14:31:19,812 INFO [StoreOpener-339879ea4311256af1be3b960184b7f5-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 339879ea4311256af1be3b960184b7f5 2024-11-26T14:31:19,813 INFO [StoreOpener-339879ea4311256af1be3b960184b7f5-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 339879ea4311256af1be3b960184b7f5 columnFamilyName a 2024-11-26T14:31:19,813 DEBUG [StoreOpener-339879ea4311256af1be3b960184b7f5-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:31:19,814 INFO [StoreOpener-339879ea4311256af1be3b960184b7f5-1 {}] regionserver.HStore(327): Store=339879ea4311256af1be3b960184b7f5/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-26T14:31:19,814 INFO [StoreOpener-339879ea4311256af1be3b960184b7f5-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 339879ea4311256af1be3b960184b7f5 2024-11-26T14:31:19,815 INFO [StoreOpener-339879ea4311256af1be3b960184b7f5-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 339879ea4311256af1be3b960184b7f5 columnFamilyName b 2024-11-26T14:31:19,815 DEBUG [StoreOpener-339879ea4311256af1be3b960184b7f5-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:31:19,816 INFO [StoreOpener-339879ea4311256af1be3b960184b7f5-1 {}] regionserver.HStore(327): Store=339879ea4311256af1be3b960184b7f5/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-26T14:31:19,816 INFO [StoreOpener-339879ea4311256af1be3b960184b7f5-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 339879ea4311256af1be3b960184b7f5 2024-11-26T14:31:19,817 INFO [StoreOpener-339879ea4311256af1be3b960184b7f5-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 339879ea4311256af1be3b960184b7f5 columnFamilyName c 2024-11-26T14:31:19,817 DEBUG [StoreOpener-339879ea4311256af1be3b960184b7f5-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:31:19,818 INFO [StoreOpener-339879ea4311256af1be3b960184b7f5-1 {}] regionserver.HStore(327): Store=339879ea4311256af1be3b960184b7f5/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-26T14:31:19,818 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 339879ea4311256af1be3b960184b7f5 2024-11-26T14:31:19,819 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44999/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/339879ea4311256af1be3b960184b7f5 2024-11-26T14:31:19,820 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44999/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/339879ea4311256af1be3b960184b7f5 2024-11-26T14:31:19,821 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 339879ea4311256af1be3b960184b7f5 2024-11-26T14:31:19,821 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 339879ea4311256af1be3b960184b7f5 2024-11-26T14:31:19,822 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testRegionMadeOfBulkLoadedFilesOnly descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-26T14:31:19,824 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 339879ea4311256af1be3b960184b7f5 2024-11-26T14:31:19,825 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 339879ea4311256af1be3b960184b7f5; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=74146881, jitterRate=0.1048746258020401}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-26T14:31:19,826 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 339879ea4311256af1be3b960184b7f5: Writing region info on filesystem at 1732631479808Initializing all the Stores at 1732631479809 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732631479809Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732631479812 (+3 ms)Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732631479812Cleaning up temporary data from old regions at 1732631479821 (+9 ms)Region opened successfully at 1732631479826 (+5 ms) 2024-11-26T14:31:19,831 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44999/hbase/testRegionMadeOfBulkLoadedFilesOnly/hfile is 28, key is \x0D/a:a/1732631479830/Put/seqid=0 2024-11-26T14:31:19,841 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36277 is added to blk_1073741881_1058 (size=4826) 2024-11-26T14:31:19,842 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36307 is added to blk_1073741881_1058 (size=4826) 2024-11-26T14:31:19,842 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38147 is added to blk_1073741881_1058 (size=4826) 2024-11-26T14:31:19,843 INFO [Time-limited test {}] regionserver.HStore(614): Validating hfile at hdfs://localhost:44999/hbase/testRegionMadeOfBulkLoadedFilesOnly/hfile for inclusion in 339879ea4311256af1be3b960184b7f5/a 2024-11-26T14:31:19,853 DEBUG [Time-limited test {}] regionserver.HStore(626): HFile bounds: first= last=z 2024-11-26T14:31:19,853 DEBUG [Time-limited test {}] regionserver.HStore(628): Region bounds: first= last= 2024-11-26T14:31:19,853 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for 339879ea4311256af1be3b960184b7f5: 2024-11-26T14:31:19,855 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44999/hbase/testRegionMadeOfBulkLoadedFilesOnly/hfile as hdfs://localhost:44999/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/339879ea4311256af1be3b960184b7f5/a/bb2283c4a8a045d8a2b88dd0b931d9a7_SeqId_3_ 2024-11-26T14:31:19,857 INFO [Time-limited test {}] regionserver.HStore(700): Loaded HFile hdfs://localhost:44999/hbase/testRegionMadeOfBulkLoadedFilesOnly/hfile into 339879ea4311256af1be3b960184b7f5/a as hdfs://localhost:44999/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/339879ea4311256af1be3b960184b7f5/a/bb2283c4a8a045d8a2b88dd0b931d9a7_SeqId_3_ - updating store file list. 2024-11-26T14:31:19,864 DEBUG [Time-limited test {}] regionserver.HStoreFile(483): HFile Bloom filter type for bb2283c4a8a045d8a2b88dd0b931d9a7_SeqId_3_: NONE, but ROW specified in column family configuration 2024-11-26T14:31:19,864 INFO [Time-limited test {}] regionserver.HStore(722): Loaded HFile hdfs://localhost:44999/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/339879ea4311256af1be3b960184b7f5/a/bb2283c4a8a045d8a2b88dd0b931d9a7_SeqId_3_ into 339879ea4311256af1be3b960184b7f5/a 2024-11-26T14:31:19,864 INFO [Time-limited test {}] regionserver.HStore(706): Successfully loaded hdfs://localhost:44999/hbase/testRegionMadeOfBulkLoadedFilesOnly/hfile into 339879ea4311256af1be3b960184b7f5/a (new location: hdfs://localhost:44999/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/339879ea4311256af1be3b960184b7f5/a/bb2283c4a8a045d8a2b88dd0b931d9a7_SeqId_3_) 2024-11-26T14:31:19,910 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:44999/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1732631479646/wal.1732631479779, size=0 (0bytes) 2024-11-26T14:31:19,910 WARN [Time-limited test {}] wal.WALSplitter(453): File hdfs://localhost:44999/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1732631479646/wal.1732631479779 might be still open, length is 0 2024-11-26T14:31:19,910 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:44999/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1732631479646/wal.1732631479779 2024-11-26T14:31:19,910 WARN [IPC Server handler 2 on default port 44999 {}] namenode.FSNamesystem(3854): DIR* NameSystem.internalReleaseLease: File /hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1732631479646/wal.1732631479779 has not been closed. Lease recovery is in progress. RecoveryId = 1059 for block blk_1073741880_1057 2024-11-26T14:31:19,911 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=0 on file=hdfs://localhost:44999/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1732631479646/wal.1732631479779 after 1ms 2024-11-26T14:31:22,847 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_2043263076_22 at /127.0.0.1:40110 [Receiving block BP-1347678837-172.17.0.3-1732631464290:blk_1073741880_1057] {}] datanode.DataXceiver(331): 127.0.0.1:36307:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:40110 dst: /127.0.0.1:36307 java.io.InterruptedIOException: Interrupted while waiting for IO on channel java.nio.channels.SocketChannel[connected local=localhost/127.0.0.1:36307 remote=/127.0.0.1:40110]. Total timeout mills is 60000, 57024 millis timeout left. at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:350) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?] at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?] at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?] at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?] at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?] at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T14:31:22,847 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_2043263076_22 at /127.0.0.1:47314 [Receiving block BP-1347678837-172.17.0.3-1732631464290:blk_1073741880_1057] {}] datanode.DataXceiver(331): 127.0.0.1:36277:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:47314 dst: /127.0.0.1:36277 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T14:31:22,847 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_2043263076_22 at /127.0.0.1:43140 [Receiving block BP-1347678837-172.17.0.3-1732631464290:blk_1073741880_1057] {}] datanode.DataXceiver(331): 127.0.0.1:38147:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:43140 dst: /127.0.0.1:38147 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T14:31:22,851 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36307 is added to blk_1073741880_1059 (size=434) 2024-11-26T14:31:22,852 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36277 is added to blk_1073741880_1059 (size=434) 2024-11-26T14:31:23,912 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=1 on file=hdfs://localhost:44999/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1732631479646/wal.1732631479779 after 4002ms 2024-11-26T14:31:23,917 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:44999/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1732631479646/wal.1732631479779: isRecoveredEdits=false, hasTagCompression=true, hasValueCompression=false, valueCompressionType=null 2024-11-26T14:31:23,918 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:44999/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1732631479646/wal.1732631479779 took 4008ms 2024-11-26T14:31:23,921 INFO [Time-limited test {}] wal.WALSplitter(493): EOF from hdfs://localhost:44999/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1732631479646/wal.1732631479779; continuing. 2024-11-26T14:31:23,921 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:44999/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1732631479646/wal.1732631479779 so closing down 2024-11-26T14:31:23,921 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-26T14:31:23,924 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000005-wal.1732631479779.temp 2024-11-26T14:31:23,926 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:44999/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/339879ea4311256af1be3b960184b7f5/recovered.edits/0000000000000000005-wal.1732631479779.temp 2024-11-26T14:31:23,926 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-11-26T14:31:23,934 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36277 is added to blk_1073741882_1060 (size=236) 2024-11-26T14:31:23,934 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38147 is added to blk_1073741882_1060 (size=236) 2024-11-26T14:31:23,935 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36307 is added to blk_1073741882_1060 (size=236) 2024-11-26T14:31:23,936 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:44999/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/339879ea4311256af1be3b960184b7f5/recovered.edits/0000000000000000005-wal.1732631479779.temp (wrote 1 edits, skipped 0 edits in 0 ms) 2024-11-26T14:31:23,938 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:44999/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/339879ea4311256af1be3b960184b7f5/recovered.edits/0000000000000000005-wal.1732631479779.temp to hdfs://localhost:44999/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/339879ea4311256af1be3b960184b7f5/recovered.edits/0000000000000000005 2024-11-26T14:31:23,938 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 2 edits across 1 Regions in 20 ms; skipped=1; WAL=hdfs://localhost:44999/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1732631479646/wal.1732631479779, size=0, length=0, corrupted=false, cancelled=false 2024-11-26T14:31:23,938 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:44999/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1732631479646/wal.1732631479779, journal: Splitting hdfs://localhost:44999/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1732631479646/wal.1732631479779, size=0 (0bytes) at 1732631479910Finishing writing output for hdfs://localhost:44999/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1732631479646/wal.1732631479779 so closing down at 1732631483921 (+4011 ms)Creating recovered edits writer path=hdfs://localhost:44999/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/339879ea4311256af1be3b960184b7f5/recovered.edits/0000000000000000005-wal.1732631479779.temp at 1732631483926 (+5 ms)3 split writer threads finished at 1732631483926Closed recovered edits writer path=hdfs://localhost:44999/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/339879ea4311256af1be3b960184b7f5/recovered.edits/0000000000000000005-wal.1732631479779.temp (wrote 1 edits, skipped 0 edits in 0 ms) at 1732631483936 (+10 ms)Rename recovered edits hdfs://localhost:44999/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/339879ea4311256af1be3b960184b7f5/recovered.edits/0000000000000000005-wal.1732631479779.temp to hdfs://localhost:44999/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/339879ea4311256af1be3b960184b7f5/recovered.edits/0000000000000000005 at 1732631483938 (+2 ms)Processed 2 edits across 1 Regions in 20 ms; skipped=1; WAL=hdfs://localhost:44999/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1732631479646/wal.1732631479779, size=0, length=0, corrupted=false, cancelled=false at 1732631483938 2024-11-26T14:31:23,940 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:44999/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1732631479646/wal.1732631479779 to hdfs://localhost:44999/hbase/oldWALs/wal.1732631479779 2024-11-26T14:31:23,941 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:44999/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/339879ea4311256af1be3b960184b7f5/recovered.edits/0000000000000000005 2024-11-26T14:31:23,942 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-26T14:31:23,944 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:44999/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1732631479646, archiveDir=hdfs://localhost:44999/hbase/oldWALs, maxLogs=32 2024-11-26T14:31:23,959 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1732631479646/wal.1732631483944, exclude list is [], retry=0 2024-11-26T14:31:23,962 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36277,DS-9cb9383d-a6c8-4ba6-83b3-6706331a6539,DISK] 2024-11-26T14:31:23,962 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36307,DS-dd37de78-b3d5-4d7c-8223-4b15101c21bf,DISK] 2024-11-26T14:31:23,962 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38147,DS-44b12d44-38d0-4c76-b56f-3ca8d8caa398,DISK] 2024-11-26T14:31:23,964 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1732631479646/wal.1732631483944 2024-11-26T14:31:23,965 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:40449:40449),(127.0.0.1/127.0.0.1:34431:34431),(127.0.0.1/127.0.0.1:35081:35081)] 2024-11-26T14:31:23,965 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 339879ea4311256af1be3b960184b7f5, NAME => 'testRegionMadeOfBulkLoadedFilesOnly,,1732631479648.339879ea4311256af1be3b960184b7f5.', STARTKEY => '', ENDKEY => ''} 2024-11-26T14:31:23,965 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testRegionMadeOfBulkLoadedFilesOnly,,1732631479648.339879ea4311256af1be3b960184b7f5.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-26T14:31:23,965 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 339879ea4311256af1be3b960184b7f5 2024-11-26T14:31:23,965 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 339879ea4311256af1be3b960184b7f5 2024-11-26T14:31:23,967 INFO [StoreOpener-339879ea4311256af1be3b960184b7f5-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 339879ea4311256af1be3b960184b7f5 2024-11-26T14:31:23,968 INFO [StoreOpener-339879ea4311256af1be3b960184b7f5-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 339879ea4311256af1be3b960184b7f5 columnFamilyName a 2024-11-26T14:31:23,968 DEBUG [StoreOpener-339879ea4311256af1be3b960184b7f5-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:31:23,974 DEBUG [StoreFileOpener-339879ea4311256af1be3b960184b7f5-a-1 {}] regionserver.HStoreFile(483): HFile Bloom filter type for bb2283c4a8a045d8a2b88dd0b931d9a7_SeqId_3_: NONE, but ROW specified in column family configuration 2024-11-26T14:31:23,975 DEBUG [StoreOpener-339879ea4311256af1be3b960184b7f5-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:44999/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/339879ea4311256af1be3b960184b7f5/a/bb2283c4a8a045d8a2b88dd0b931d9a7_SeqId_3_ 2024-11-26T14:31:23,975 INFO [StoreOpener-339879ea4311256af1be3b960184b7f5-1 {}] regionserver.HStore(327): Store=339879ea4311256af1be3b960184b7f5/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-26T14:31:23,975 INFO [StoreOpener-339879ea4311256af1be3b960184b7f5-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 339879ea4311256af1be3b960184b7f5 2024-11-26T14:31:23,976 INFO [StoreOpener-339879ea4311256af1be3b960184b7f5-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 339879ea4311256af1be3b960184b7f5 columnFamilyName b 2024-11-26T14:31:23,976 DEBUG [StoreOpener-339879ea4311256af1be3b960184b7f5-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:31:23,977 INFO [StoreOpener-339879ea4311256af1be3b960184b7f5-1 {}] regionserver.HStore(327): Store=339879ea4311256af1be3b960184b7f5/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-26T14:31:23,977 INFO [StoreOpener-339879ea4311256af1be3b960184b7f5-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 339879ea4311256af1be3b960184b7f5 2024-11-26T14:31:23,978 INFO [StoreOpener-339879ea4311256af1be3b960184b7f5-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 339879ea4311256af1be3b960184b7f5 columnFamilyName c 2024-11-26T14:31:23,978 DEBUG [StoreOpener-339879ea4311256af1be3b960184b7f5-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:31:23,978 INFO [StoreOpener-339879ea4311256af1be3b960184b7f5-1 {}] regionserver.HStore(327): Store=339879ea4311256af1be3b960184b7f5/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-26T14:31:23,978 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 339879ea4311256af1be3b960184b7f5 2024-11-26T14:31:23,979 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44999/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/339879ea4311256af1be3b960184b7f5 2024-11-26T14:31:23,982 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:44999/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/339879ea4311256af1be3b960184b7f5 2024-11-26T14:31:23,982 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:44999/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/339879ea4311256af1be3b960184b7f5/recovered.edits/0000000000000000005 2024-11-26T14:31:23,985 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:44999/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/339879ea4311256af1be3b960184b7f5/recovered.edits/0000000000000000005: isRecoveredEdits=true, hasTagCompression=true, hasValueCompression=false, valueCompressionType=null 2024-11-26T14:31:23,986 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 1, skipped 0, firstSequenceIdInLog=5, maxSequenceIdInLog=5, path=hdfs://localhost:44999/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/339879ea4311256af1be3b960184b7f5/recovered.edits/0000000000000000005 2024-11-26T14:31:23,986 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 339879ea4311256af1be3b960184b7f5 3/3 column families, dataSize=58 B heapSize=904 B 2024-11-26T14:31:24,000 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44999/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/339879ea4311256af1be3b960184b7f5/.tmp/a/1da0df16c1e344899994f391d09bacdf is 62, key is testRegionMadeOfBulkLoadedFilesOnly/a:a/1732631479870/Put/seqid=0 2024-11-26T14:31:24,006 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36277 is added to blk_1073741884_1062 (size=5149) 2024-11-26T14:31:24,006 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36307 is added to blk_1073741884_1062 (size=5149) 2024-11-26T14:31:24,007 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38147 is added to blk_1073741884_1062 (size=5149) 2024-11-26T14:31:24,007 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=58 B at sequenceid=5 (bloomFilter=true), to=hdfs://localhost:44999/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/339879ea4311256af1be3b960184b7f5/.tmp/a/1da0df16c1e344899994f391d09bacdf 2024-11-26T14:31:24,013 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44999/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/339879ea4311256af1be3b960184b7f5/.tmp/a/1da0df16c1e344899994f391d09bacdf as hdfs://localhost:44999/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/339879ea4311256af1be3b960184b7f5/a/1da0df16c1e344899994f391d09bacdf 2024-11-26T14:31:24,019 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44999/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/339879ea4311256af1be3b960184b7f5/a/1da0df16c1e344899994f391d09bacdf, entries=1, sequenceid=5, filesize=5.0 K 2024-11-26T14:31:24,020 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~58 B/58, heapSize ~376 B/376, currentSize=0 B/0 for 339879ea4311256af1be3b960184b7f5 in 33ms, sequenceid=5, compaction requested=false; wal=null 2024-11-26T14:31:24,020 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:44999/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/339879ea4311256af1be3b960184b7f5/recovered.edits/0000000000000000005 2024-11-26T14:31:24,022 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 339879ea4311256af1be3b960184b7f5 2024-11-26T14:31:24,022 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 339879ea4311256af1be3b960184b7f5 2024-11-26T14:31:24,023 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testRegionMadeOfBulkLoadedFilesOnly descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-26T14:31:24,024 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 339879ea4311256af1be3b960184b7f5 2024-11-26T14:31:24,027 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44999/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/339879ea4311256af1be3b960184b7f5/recovered.edits/5.seqid, newMaxSeqId=5, maxSeqId=1 2024-11-26T14:31:24,028 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 339879ea4311256af1be3b960184b7f5; next sequenceid=6; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=61324801, jitterRate=-0.08618925511837006}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-26T14:31:24,028 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 339879ea4311256af1be3b960184b7f5: Writing region info on filesystem at 1732631483965Initializing all the Stores at 1732631483966 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732631483966Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732631483967 (+1 ms)Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732631483967Obtaining lock to block concurrent updates at 1732631483986 (+19 ms)Preparing flush snapshotting stores in 339879ea4311256af1be3b960184b7f5 at 1732631483986Finished memstore snapshotting testRegionMadeOfBulkLoadedFilesOnly,,1732631479648.339879ea4311256af1be3b960184b7f5., syncing WAL and waiting on mvcc, flushsize=dataSize=58, getHeapSize=856, getOffHeapSize=0, getCellsCount=1 at 1732631483986Flushing stores of testRegionMadeOfBulkLoadedFilesOnly,,1732631479648.339879ea4311256af1be3b960184b7f5. at 1732631483986Flushing 339879ea4311256af1be3b960184b7f5/a: creating writer at 1732631483986Flushing 339879ea4311256af1be3b960184b7f5/a: appending metadata at 1732631483999 (+13 ms)Flushing 339879ea4311256af1be3b960184b7f5/a: closing flushed file at 1732631483999Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@79c80a1d: reopening flushed file at 1732631484012 (+13 ms)Finished flush of dataSize ~58 B/58, heapSize ~376 B/376, currentSize=0 B/0 for 339879ea4311256af1be3b960184b7f5 in 33ms, sequenceid=5, compaction requested=false; wal=null at 1732631484020 (+8 ms)Cleaning up temporary data from old regions at 1732631484022 (+2 ms)Region opened successfully at 1732631484028 (+6 ms) 2024-11-26T14:31:24,032 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 339879ea4311256af1be3b960184b7f5, disabling compactions & flushes 2024-11-26T14:31:24,033 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testRegionMadeOfBulkLoadedFilesOnly,,1732631479648.339879ea4311256af1be3b960184b7f5. 2024-11-26T14:31:24,033 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testRegionMadeOfBulkLoadedFilesOnly,,1732631479648.339879ea4311256af1be3b960184b7f5. 2024-11-26T14:31:24,033 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testRegionMadeOfBulkLoadedFilesOnly,,1732631479648.339879ea4311256af1be3b960184b7f5. after waiting 0 ms 2024-11-26T14:31:24,033 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testRegionMadeOfBulkLoadedFilesOnly,,1732631479648.339879ea4311256af1be3b960184b7f5. 2024-11-26T14:31:24,034 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testRegionMadeOfBulkLoadedFilesOnly,,1732631479648.339879ea4311256af1be3b960184b7f5. 2024-11-26T14:31:24,034 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 339879ea4311256af1be3b960184b7f5: Waiting for close lock at 1732631484032Disabling compacts and flushes for region at 1732631484032Disabling writes for close at 1732631484033 (+1 ms)Writing region close event to WAL at 1732631484034 (+1 ms)Closed at 1732631484034 2024-11-26T14:31:24,037 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36307 is added to blk_1073741883_1061 (size=93) 2024-11-26T14:31:24,037 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38147 is added to blk_1073741883_1061 (size=93) 2024-11-26T14:31:24,037 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36277 is added to blk_1073741883_1061 (size=93) 2024-11-26T14:31:24,039 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-11-26T14:31:24,039 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL wal:(num 1732631483944) 2024-11-26T14:31:24,053 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestAsyncWALReplayCompressed#testRegionMadeOfBulkLoadedFilesOnly Thread=416 (was 409) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/175d8e9e-4f50-9d71-fd80-148275c72391/cluster_49e351ba-434f-d14a-4712-fb0fcff718bc/data/data6 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: LeaseRenewer:jenkinstestRegionMadeOfBulkLoadedFilesOnly@localhost:44999 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-17-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-17-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1818460696_22 at /127.0.0.1:39498 [Waiting for operation #3] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1818460696_22 at /127.0.0.1:39130 [Waiting for operation #9] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Client (1987708253) connection to localhost/127.0.0.1:44999 from jenkinstestRegionMadeOfBulkLoadedFilesOnly java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) Potentially hanging thread: AsyncFSWAL-17-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/175d8e9e-4f50-9d71-fd80-148275c72391/cluster_49e351ba-434f-d14a-4712-fb0fcff718bc/data/data2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1818460696_22 at /127.0.0.1:43760 [Waiting for operation #5] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/175d8e9e-4f50-9d71-fd80-148275c72391/cluster_49e351ba-434f-d14a-4712-fb0fcff718bc/data/data5 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/175d8e9e-4f50-9d71-fd80-148275c72391/cluster_49e351ba-434f-d14a-4712-fb0fcff718bc/data/data1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=1023 (was 963) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=248 (was 261), ProcessCount=11 (was 11), AvailableMemoryMB=9088 (was 9152) 2024-11-26T14:31:24,066 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestAsyncWALReplayCompressed#testReplayEditsAfterRegionMovedWithMultiCF Thread=416, OpenFileDescriptor=1023, MaxFileDescriptor=1048576, SystemLoadAverage=248, ProcessCount=11, AvailableMemoryMB=9088 2024-11-26T14:31:24,080 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-26T14:31:24,087 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-26T14:31:24,091 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] client.AsyncConnectionImpl(321): The fetched master address is 19945b63c2b4,43577,1732631468672 2024-11-26T14:31:24,093 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@78289f12 2024-11-26T14:31:24,094 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-26T14:31:24,096 INFO [HMaster-EventLoopGroup-2-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:56494, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-26T14:31:24,099 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43577 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.3 create 'testReplayEditsAfterRegionMovedWithMultiCF', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'cf1', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'cf2', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-26T14:31:24,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43577 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=testReplayEditsAfterRegionMovedWithMultiCF 2024-11-26T14:31:24,108 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=testReplayEditsAfterRegionMovedWithMultiCF execute state=CREATE_TABLE_PRE_OPERATION 2024-11-26T14:31:24,110 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43577 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.3 procedure request for creating table: namespace: "default" qualifier: "testReplayEditsAfterRegionMovedWithMultiCF" procId is: 4 2024-11-26T14:31:24,110 DEBUG [PEWorker-2 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:31:24,112 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=testReplayEditsAfterRegionMovedWithMultiCF execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-26T14:31:24,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43577 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-26T14:31:24,123 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38147 is added to blk_1073741885_1063 (size=694) 2024-11-26T14:31:24,123 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36277 is added to blk_1073741885_1063 (size=694) 2024-11-26T14:31:24,124 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36307 is added to blk_1073741885_1063 (size=694) 2024-11-26T14:31:24,126 INFO [RegionOpenAndInit-testReplayEditsAfterRegionMovedWithMultiCF-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 40329a66ba4e990f8463ecffc5dfdaa9, NAME => 'testReplayEditsAfterRegionMovedWithMultiCF,,1732631484097.40329a66ba4e990f8463ecffc5dfdaa9.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testReplayEditsAfterRegionMovedWithMultiCF', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'cf1', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'cf2', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:44999/user/jenkins/test-data/2de93217-0593-bca0-5c5f-879b6362c76f 2024-11-26T14:31:24,135 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36307 is added to blk_1073741886_1064 (size=77) 2024-11-26T14:31:24,135 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38147 is added to blk_1073741886_1064 (size=77) 2024-11-26T14:31:24,135 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36277 is added to blk_1073741886_1064 (size=77) 2024-11-26T14:31:24,137 DEBUG [RegionOpenAndInit-testReplayEditsAfterRegionMovedWithMultiCF-pool-0 {}] regionserver.HRegion(898): Instantiated testReplayEditsAfterRegionMovedWithMultiCF,,1732631484097.40329a66ba4e990f8463ecffc5dfdaa9.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-26T14:31:24,137 DEBUG [RegionOpenAndInit-testReplayEditsAfterRegionMovedWithMultiCF-pool-0 {}] regionserver.HRegion(1722): Closing 40329a66ba4e990f8463ecffc5dfdaa9, disabling compactions & flushes 2024-11-26T14:31:24,137 INFO [RegionOpenAndInit-testReplayEditsAfterRegionMovedWithMultiCF-pool-0 {}] regionserver.HRegion(1755): Closing region testReplayEditsAfterRegionMovedWithMultiCF,,1732631484097.40329a66ba4e990f8463ecffc5dfdaa9. 2024-11-26T14:31:24,137 DEBUG [RegionOpenAndInit-testReplayEditsAfterRegionMovedWithMultiCF-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1732631484097.40329a66ba4e990f8463ecffc5dfdaa9. 2024-11-26T14:31:24,137 DEBUG [RegionOpenAndInit-testReplayEditsAfterRegionMovedWithMultiCF-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1732631484097.40329a66ba4e990f8463ecffc5dfdaa9. after waiting 0 ms 2024-11-26T14:31:24,137 DEBUG [RegionOpenAndInit-testReplayEditsAfterRegionMovedWithMultiCF-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsAfterRegionMovedWithMultiCF,,1732631484097.40329a66ba4e990f8463ecffc5dfdaa9. 2024-11-26T14:31:24,137 INFO [RegionOpenAndInit-testReplayEditsAfterRegionMovedWithMultiCF-pool-0 {}] regionserver.HRegion(1973): Closed testReplayEditsAfterRegionMovedWithMultiCF,,1732631484097.40329a66ba4e990f8463ecffc5dfdaa9. 2024-11-26T14:31:24,137 DEBUG [RegionOpenAndInit-testReplayEditsAfterRegionMovedWithMultiCF-pool-0 {}] regionserver.HRegion(1676): Region close journal for 40329a66ba4e990f8463ecffc5dfdaa9: Waiting for close lock at 1732631484137Disabling compacts and flushes for region at 1732631484137Disabling writes for close at 1732631484137Writing region close event to WAL at 1732631484137Closed at 1732631484137 2024-11-26T14:31:24,139 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=testReplayEditsAfterRegionMovedWithMultiCF execute state=CREATE_TABLE_ADD_TO_META 2024-11-26T14:31:24,144 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testReplayEditsAfterRegionMovedWithMultiCF,,1732631484097.40329a66ba4e990f8463ecffc5dfdaa9.","families":{"info":[{"qualifier":"regioninfo","vlen":76,"tag":[],"timestamp":"1732631484139"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732631484139"}]},"ts":"1732631484139"} 2024-11-26T14:31:24,147 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-26T14:31:24,149 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=testReplayEditsAfterRegionMovedWithMultiCF execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-26T14:31:24,152 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testReplayEditsAfterRegionMovedWithMultiCF","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732631484149"}]},"ts":"1732631484149"} 2024-11-26T14:31:24,155 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=testReplayEditsAfterRegionMovedWithMultiCF, state=ENABLING in hbase:meta 2024-11-26T14:31:24,156 DEBUG [PEWorker-2 {}] balancer.BalancerClusterState(204): Hosts are {19945b63c2b4=0} racks are {/default-rack=0} 2024-11-26T14:31:24,158 DEBUG [PEWorker-2 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-11-26T14:31:24,158 DEBUG [PEWorker-2 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-11-26T14:31:24,158 DEBUG [PEWorker-2 {}] balancer.BalancerClusterState(303): server 2 has 0 regions 2024-11-26T14:31:24,158 DEBUG [PEWorker-2 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-11-26T14:31:24,158 DEBUG [PEWorker-2 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-11-26T14:31:24,158 DEBUG [PEWorker-2 {}] balancer.BalancerClusterState(310): server 2 is on host 0 2024-11-26T14:31:24,158 INFO [PEWorker-2 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-11-26T14:31:24,158 INFO [PEWorker-2 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-11-26T14:31:24,158 INFO [PEWorker-2 {}] balancer.BalancerClusterState(321): server 2 is on rack 0 2024-11-26T14:31:24,158 DEBUG [PEWorker-2 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-11-26T14:31:24,160 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=40329a66ba4e990f8463ecffc5dfdaa9, ASSIGN}] 2024-11-26T14:31:24,162 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=40329a66ba4e990f8463ecffc5dfdaa9, ASSIGN 2024-11-26T14:31:24,163 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=40329a66ba4e990f8463ecffc5dfdaa9, ASSIGN; state=OFFLINE, location=19945b63c2b4,35419,1732631469384; forceNewPlan=false, retain=false 2024-11-26T14:31:24,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43577 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-26T14:31:24,317 INFO [19945b63c2b4:43577 {}] balancer.BaseLoadBalancer(388): Reassigned 1 regions. 1 retained the pre-restart assignment. 2024-11-26T14:31:24,318 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=40329a66ba4e990f8463ecffc5dfdaa9, regionState=OPENING, regionLocation=19945b63c2b4,35419,1732631469384 2024-11-26T14:31:24,323 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=40329a66ba4e990f8463ecffc5dfdaa9, ASSIGN because future has completed 2024-11-26T14:31:24,325 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 40329a66ba4e990f8463ecffc5dfdaa9, server=19945b63c2b4,35419,1732631469384}] 2024-11-26T14:31:24,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43577 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-26T14:31:24,480 DEBUG [RSProcedureDispatcher-pool-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-26T14:31:24,482 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:50213, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-26T14:31:24,487 INFO [RS_OPEN_REGION-regionserver/19945b63c2b4:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(132): Open testReplayEditsAfterRegionMovedWithMultiCF,,1732631484097.40329a66ba4e990f8463ecffc5dfdaa9. 2024-11-26T14:31:24,488 DEBUG [RS_OPEN_REGION-regionserver/19945b63c2b4:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7752): Opening region: {ENCODED => 40329a66ba4e990f8463ecffc5dfdaa9, NAME => 'testReplayEditsAfterRegionMovedWithMultiCF,,1732631484097.40329a66ba4e990f8463ecffc5dfdaa9.', STARTKEY => '', ENDKEY => ''} 2024-11-26T14:31:24,488 DEBUG [RS_OPEN_REGION-regionserver/19945b63c2b4:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testReplayEditsAfterRegionMovedWithMultiCF 40329a66ba4e990f8463ecffc5dfdaa9 2024-11-26T14:31:24,488 DEBUG [RS_OPEN_REGION-regionserver/19945b63c2b4:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(898): Instantiated testReplayEditsAfterRegionMovedWithMultiCF,,1732631484097.40329a66ba4e990f8463ecffc5dfdaa9.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-26T14:31:24,489 DEBUG [RS_OPEN_REGION-regionserver/19945b63c2b4:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7794): checking encryption for 40329a66ba4e990f8463ecffc5dfdaa9 2024-11-26T14:31:24,489 DEBUG [RS_OPEN_REGION-regionserver/19945b63c2b4:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7797): checking classloading for 40329a66ba4e990f8463ecffc5dfdaa9 2024-11-26T14:31:24,491 INFO [StoreOpener-40329a66ba4e990f8463ecffc5dfdaa9-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf1 of region 40329a66ba4e990f8463ecffc5dfdaa9 2024-11-26T14:31:24,494 INFO [StoreOpener-40329a66ba4e990f8463ecffc5dfdaa9-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 40329a66ba4e990f8463ecffc5dfdaa9 columnFamilyName cf1 2024-11-26T14:31:24,494 DEBUG [StoreOpener-40329a66ba4e990f8463ecffc5dfdaa9-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:31:24,495 INFO [StoreOpener-40329a66ba4e990f8463ecffc5dfdaa9-1 {}] regionserver.HStore(327): Store=40329a66ba4e990f8463ecffc5dfdaa9/cf1, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-26T14:31:24,495 INFO [StoreOpener-40329a66ba4e990f8463ecffc5dfdaa9-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf2 of region 40329a66ba4e990f8463ecffc5dfdaa9 2024-11-26T14:31:24,497 INFO [StoreOpener-40329a66ba4e990f8463ecffc5dfdaa9-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 40329a66ba4e990f8463ecffc5dfdaa9 columnFamilyName cf2 2024-11-26T14:31:24,497 DEBUG [StoreOpener-40329a66ba4e990f8463ecffc5dfdaa9-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:31:24,497 INFO [StoreOpener-40329a66ba4e990f8463ecffc5dfdaa9-1 {}] regionserver.HStore(327): Store=40329a66ba4e990f8463ecffc5dfdaa9/cf2, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-26T14:31:24,497 DEBUG [RS_OPEN_REGION-regionserver/19945b63c2b4:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1038): replaying wal for 40329a66ba4e990f8463ecffc5dfdaa9 2024-11-26T14:31:24,498 DEBUG [RS_OPEN_REGION-regionserver/19945b63c2b4:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44999/user/jenkins/test-data/2de93217-0593-bca0-5c5f-879b6362c76f/data/default/testReplayEditsAfterRegionMovedWithMultiCF/40329a66ba4e990f8463ecffc5dfdaa9 2024-11-26T14:31:24,499 DEBUG [RS_OPEN_REGION-regionserver/19945b63c2b4:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44999/user/jenkins/test-data/2de93217-0593-bca0-5c5f-879b6362c76f/data/default/testReplayEditsAfterRegionMovedWithMultiCF/40329a66ba4e990f8463ecffc5dfdaa9 2024-11-26T14:31:24,499 DEBUG [RS_OPEN_REGION-regionserver/19945b63c2b4:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1048): stopping wal replay for 40329a66ba4e990f8463ecffc5dfdaa9 2024-11-26T14:31:24,499 DEBUG [RS_OPEN_REGION-regionserver/19945b63c2b4:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1060): Cleaning up temporary data for 40329a66ba4e990f8463ecffc5dfdaa9 2024-11-26T14:31:24,500 DEBUG [RS_OPEN_REGION-regionserver/19945b63c2b4:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsAfterRegionMovedWithMultiCF descriptor;using region.getMemStoreFlushHeapSize/# of families (64.0 M)) instead. 2024-11-26T14:31:24,502 DEBUG [RS_OPEN_REGION-regionserver/19945b63c2b4:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1093): writing seq id for 40329a66ba4e990f8463ecffc5dfdaa9 2024-11-26T14:31:24,504 DEBUG [RS_OPEN_REGION-regionserver/19945b63c2b4:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44999/user/jenkins/test-data/2de93217-0593-bca0-5c5f-879b6362c76f/data/default/testReplayEditsAfterRegionMovedWithMultiCF/40329a66ba4e990f8463ecffc5dfdaa9/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-26T14:31:24,504 INFO [RS_OPEN_REGION-regionserver/19945b63c2b4:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1114): Opened 40329a66ba4e990f8463ecffc5dfdaa9; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=74272682, jitterRate=0.10674920678138733}}}, FlushLargeStoresPolicy{flushSizeLowerBound=67108864} 2024-11-26T14:31:24,504 DEBUG [RS_OPEN_REGION-regionserver/19945b63c2b4:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 40329a66ba4e990f8463ecffc5dfdaa9 2024-11-26T14:31:24,505 DEBUG [RS_OPEN_REGION-regionserver/19945b63c2b4:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1006): Region open journal for 40329a66ba4e990f8463ecffc5dfdaa9: Running coprocessor pre-open hook at 1732631484489Writing region info on filesystem at 1732631484489Initializing all the Stores at 1732631484491 (+2 ms)Instantiating store for column family {NAME => 'cf1', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732631484491Instantiating store for column family {NAME => 'cf2', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732631484491Cleaning up temporary data from old regions at 1732631484499 (+8 ms)Running coprocessor post-open hooks at 1732631484505 (+6 ms)Region opened successfully at 1732631484505 2024-11-26T14:31:24,507 INFO [RS_OPEN_REGION-regionserver/19945b63c2b4:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2236): Post open deploy tasks for testReplayEditsAfterRegionMovedWithMultiCF,,1732631484097.40329a66ba4e990f8463ecffc5dfdaa9., pid=6, masterSystemTime=1732631484480 2024-11-26T14:31:24,510 DEBUG [RS_OPEN_REGION-regionserver/19945b63c2b4:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2266): Finished post open deploy task for testReplayEditsAfterRegionMovedWithMultiCF,,1732631484097.40329a66ba4e990f8463ecffc5dfdaa9. 2024-11-26T14:31:24,510 INFO [RS_OPEN_REGION-regionserver/19945b63c2b4:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(153): Opened testReplayEditsAfterRegionMovedWithMultiCF,,1732631484097.40329a66ba4e990f8463ecffc5dfdaa9. 2024-11-26T14:31:24,510 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=40329a66ba4e990f8463ecffc5dfdaa9, regionState=OPEN, openSeqNum=2, regionLocation=19945b63c2b4,35419,1732631469384 2024-11-26T14:31:24,513 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 40329a66ba4e990f8463ecffc5dfdaa9, server=19945b63c2b4,35419,1732631469384 because future has completed 2024-11-26T14:31:24,517 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=6, resume processing ppid=5 2024-11-26T14:31:24,518 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure 40329a66ba4e990f8463ecffc5dfdaa9, server=19945b63c2b4,35419,1732631469384 in 189 msec 2024-11-26T14:31:24,520 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=5, resume processing ppid=4 2024-11-26T14:31:24,520 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=40329a66ba4e990f8463ecffc5dfdaa9, ASSIGN in 357 msec 2024-11-26T14:31:24,521 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=testReplayEditsAfterRegionMovedWithMultiCF execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-26T14:31:24,522 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testReplayEditsAfterRegionMovedWithMultiCF","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732631484521"}]},"ts":"1732631484521"} 2024-11-26T14:31:24,525 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=testReplayEditsAfterRegionMovedWithMultiCF, state=ENABLED in hbase:meta 2024-11-26T14:31:24,526 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=testReplayEditsAfterRegionMovedWithMultiCF execute state=CREATE_TABLE_POST_OPERATION 2024-11-26T14:31:24,529 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=testReplayEditsAfterRegionMovedWithMultiCF in 425 msec 2024-11-26T14:31:24,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43577 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-26T14:31:24,748 INFO [RPCClient-NioEventLoopGroup-6-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:testReplayEditsAfterRegionMovedWithMultiCF completed 2024-11-26T14:31:24,748 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table testReplayEditsAfterRegionMovedWithMultiCF get assigned. Timeout = 60000ms 2024-11-26T14:31:24,751 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-26T14:31:24,760 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table testReplayEditsAfterRegionMovedWithMultiCF assigned to meta. Checking AM states. 2024-11-26T14:31:24,761 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-26T14:31:24,762 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table testReplayEditsAfterRegionMovedWithMultiCF assigned. 2024-11-26T14:31:24,777 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testReplayEditsAfterRegionMovedWithMultiCF', row='r1', locateType=CURRENT is [region=testReplayEditsAfterRegionMovedWithMultiCF,,1732631484097.40329a66ba4e990f8463ecffc5dfdaa9., hostname=19945b63c2b4,35419,1732631469384, seqNum=2] 2024-11-26T14:31:24,778 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-26T14:31:24,781 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:47654, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-26T14:31:24,796 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43577 {}] master.HMaster(2410): Client=jenkins//172.17.0.3 move hri=40329a66ba4e990f8463ecffc5dfdaa9, source=19945b63c2b4,35419,1732631469384, destination=19945b63c2b4,40179,1732631469496, warming up region on 19945b63c2b4,40179,1732631469496 2024-11-26T14:31:24,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43577 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-26T14:31:24,800 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43577 {}] master.HMaster(2414): Client=jenkins//172.17.0.3 move hri=40329a66ba4e990f8463ecffc5dfdaa9, source=19945b63c2b4,35419,1732631469384, destination=19945b63c2b4,40179,1732631469496, running balancer 2024-11-26T14:31:24,801 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-4-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:57035, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-26T14:31:24,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43577 {}] procedure2.ProcedureExecutor(1139): Stored pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=40329a66ba4e990f8463ecffc5dfdaa9, REOPEN/MOVE 2024-11-26T14:31:24,802 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=40329a66ba4e990f8463ecffc5dfdaa9, REOPEN/MOVE 2024-11-26T14:31:24,804 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=40329a66ba4e990f8463ecffc5dfdaa9, regionState=CLOSING, regionLocation=19945b63c2b4,35419,1732631469384 2024-11-26T14:31:24,806 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40179 {}] regionserver.RSRpcServices(2066): Warmup testReplayEditsAfterRegionMovedWithMultiCF,,1732631484097.40329a66ba4e990f8463ecffc5dfdaa9. 2024-11-26T14:31:24,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40179 {}] regionserver.HRegion(7855): Warmup {ENCODED => 40329a66ba4e990f8463ecffc5dfdaa9, NAME => 'testReplayEditsAfterRegionMovedWithMultiCF,,1732631484097.40329a66ba4e990f8463ecffc5dfdaa9.', STARTKEY => '', ENDKEY => ''} 2024-11-26T14:31:24,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40179 {}] regionserver.HRegion(898): Instantiated testReplayEditsAfterRegionMovedWithMultiCF,,1732631484097.40329a66ba4e990f8463ecffc5dfdaa9.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-26T14:31:24,807 INFO [StoreOpener-40329a66ba4e990f8463ecffc5dfdaa9-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf1 of region 40329a66ba4e990f8463ecffc5dfdaa9 2024-11-26T14:31:24,808 INFO [StoreOpener-40329a66ba4e990f8463ecffc5dfdaa9-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 40329a66ba4e990f8463ecffc5dfdaa9 columnFamilyName cf1 2024-11-26T14:31:24,808 DEBUG [StoreOpener-40329a66ba4e990f8463ecffc5dfdaa9-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:31:24,808 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=40329a66ba4e990f8463ecffc5dfdaa9, REOPEN/MOVE because future has completed 2024-11-26T14:31:24,809 INFO [StoreOpener-40329a66ba4e990f8463ecffc5dfdaa9-1 {}] regionserver.HStore(327): Store=40329a66ba4e990f8463ecffc5dfdaa9/cf1, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-26T14:31:24,809 DEBUG [PEWorker-2 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-26T14:31:24,809 INFO [StoreOpener-40329a66ba4e990f8463ecffc5dfdaa9-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf2 of region 40329a66ba4e990f8463ecffc5dfdaa9 2024-11-26T14:31:24,809 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=8, ppid=7, state=RUNNABLE, hasLock=false; CloseRegionProcedure 40329a66ba4e990f8463ecffc5dfdaa9, server=19945b63c2b4,35419,1732631469384}] 2024-11-26T14:31:24,811 INFO [StoreOpener-40329a66ba4e990f8463ecffc5dfdaa9-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 40329a66ba4e990f8463ecffc5dfdaa9 columnFamilyName cf2 2024-11-26T14:31:24,811 DEBUG [StoreOpener-40329a66ba4e990f8463ecffc5dfdaa9-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:31:24,811 INFO [StoreOpener-40329a66ba4e990f8463ecffc5dfdaa9-1 {}] regionserver.HStore(327): Store=40329a66ba4e990f8463ecffc5dfdaa9/cf2, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-26T14:31:24,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40179 {}] regionserver.HRegion(1722): Closing 40329a66ba4e990f8463ecffc5dfdaa9, disabling compactions & flushes 2024-11-26T14:31:24,812 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40179 {}] regionserver.HRegion(1755): Closing region testReplayEditsAfterRegionMovedWithMultiCF,,1732631484097.40329a66ba4e990f8463ecffc5dfdaa9. 2024-11-26T14:31:24,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40179 {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1732631484097.40329a66ba4e990f8463ecffc5dfdaa9. 2024-11-26T14:31:24,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40179 {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1732631484097.40329a66ba4e990f8463ecffc5dfdaa9. after waiting 0 ms 2024-11-26T14:31:24,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40179 {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsAfterRegionMovedWithMultiCF,,1732631484097.40329a66ba4e990f8463ecffc5dfdaa9. 2024-11-26T14:31:24,812 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40179 {}] regionserver.HRegion(1973): Closed testReplayEditsAfterRegionMovedWithMultiCF,,1732631484097.40329a66ba4e990f8463ecffc5dfdaa9. 2024-11-26T14:31:24,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40179 {}] regionserver.HRegion(1676): Region close journal for 40329a66ba4e990f8463ecffc5dfdaa9: Waiting for close lock at 1732631484812Disabling compacts and flushes for region at 1732631484812Disabling writes for close at 1732631484812Writing region close event to WAL at 1732631484812Closed at 1732631484812 2024-11-26T14:31:24,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43577 {}] procedure.ProcedureSyncWait(219): waitFor pid=7 2024-11-26T14:31:24,969 INFO [RS_CLOSE_REGION-regionserver/19945b63c2b4:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] handler.UnassignRegionHandler(122): Close 40329a66ba4e990f8463ecffc5dfdaa9 2024-11-26T14:31:24,969 DEBUG [RS_CLOSE_REGION-regionserver/19945b63c2b4:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-26T14:31:24,970 DEBUG [RS_CLOSE_REGION-regionserver/19945b63c2b4:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1722): Closing 40329a66ba4e990f8463ecffc5dfdaa9, disabling compactions & flushes 2024-11-26T14:31:24,970 INFO [RS_CLOSE_REGION-regionserver/19945b63c2b4:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1755): Closing region testReplayEditsAfterRegionMovedWithMultiCF,,1732631484097.40329a66ba4e990f8463ecffc5dfdaa9. 2024-11-26T14:31:24,970 DEBUG [RS_CLOSE_REGION-regionserver/19945b63c2b4:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1732631484097.40329a66ba4e990f8463ecffc5dfdaa9. 2024-11-26T14:31:24,970 DEBUG [RS_CLOSE_REGION-regionserver/19945b63c2b4:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1732631484097.40329a66ba4e990f8463ecffc5dfdaa9. after waiting 0 ms 2024-11-26T14:31:24,970 DEBUG [RS_CLOSE_REGION-regionserver/19945b63c2b4:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsAfterRegionMovedWithMultiCF,,1732631484097.40329a66ba4e990f8463ecffc5dfdaa9. 2024-11-26T14:31:24,970 INFO [RS_CLOSE_REGION-regionserver/19945b63c2b4:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(2902): Flushing 40329a66ba4e990f8463ecffc5dfdaa9 2/2 column families, dataSize=31 B heapSize=616 B 2024-11-26T14:31:24,989 DEBUG [RS_CLOSE_REGION-regionserver/19945b63c2b4:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44999/user/jenkins/test-data/2de93217-0593-bca0-5c5f-879b6362c76f/data/default/testReplayEditsAfterRegionMovedWithMultiCF/40329a66ba4e990f8463ecffc5dfdaa9/.tmp/cf1/6b2bb3a954f34f899f665e29662182d0 is 35, key is r1/cf1:q/1732631484781/Put/seqid=0 2024-11-26T14:31:24,995 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38147 is added to blk_1073741887_1065 (size=4783) 2024-11-26T14:31:24,996 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36307 is added to blk_1073741887_1065 (size=4783) 2024-11-26T14:31:24,996 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36277 is added to blk_1073741887_1065 (size=4783) 2024-11-26T14:31:24,996 INFO [RS_CLOSE_REGION-regionserver/19945b63c2b4:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31 B at sequenceid=5 (bloomFilter=false), to=hdfs://localhost:44999/user/jenkins/test-data/2de93217-0593-bca0-5c5f-879b6362c76f/data/default/testReplayEditsAfterRegionMovedWithMultiCF/40329a66ba4e990f8463ecffc5dfdaa9/.tmp/cf1/6b2bb3a954f34f899f665e29662182d0 2024-11-26T14:31:25,004 DEBUG [RS_CLOSE_REGION-regionserver/19945b63c2b4:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44999/user/jenkins/test-data/2de93217-0593-bca0-5c5f-879b6362c76f/data/default/testReplayEditsAfterRegionMovedWithMultiCF/40329a66ba4e990f8463ecffc5dfdaa9/.tmp/cf1/6b2bb3a954f34f899f665e29662182d0 as hdfs://localhost:44999/user/jenkins/test-data/2de93217-0593-bca0-5c5f-879b6362c76f/data/default/testReplayEditsAfterRegionMovedWithMultiCF/40329a66ba4e990f8463ecffc5dfdaa9/cf1/6b2bb3a954f34f899f665e29662182d0 2024-11-26T14:31:25,011 INFO [RS_CLOSE_REGION-regionserver/19945b63c2b4:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44999/user/jenkins/test-data/2de93217-0593-bca0-5c5f-879b6362c76f/data/default/testReplayEditsAfterRegionMovedWithMultiCF/40329a66ba4e990f8463ecffc5dfdaa9/cf1/6b2bb3a954f34f899f665e29662182d0, entries=1, sequenceid=5, filesize=4.7 K 2024-11-26T14:31:25,012 INFO [RS_CLOSE_REGION-regionserver/19945b63c2b4:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(3140): Finished flush of dataSize ~31 B/31, heapSize ~344 B/344, currentSize=0 B/0 for 40329a66ba4e990f8463ecffc5dfdaa9 in 42ms, sequenceid=5, compaction requested=false 2024-11-26T14:31:25,012 DEBUG [RS_CLOSE_REGION-regionserver/19945b63c2b4:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'testReplayEditsAfterRegionMovedWithMultiCF' 2024-11-26T14:31:25,017 DEBUG [RS_CLOSE_REGION-regionserver/19945b63c2b4:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44999/user/jenkins/test-data/2de93217-0593-bca0-5c5f-879b6362c76f/data/default/testReplayEditsAfterRegionMovedWithMultiCF/40329a66ba4e990f8463ecffc5dfdaa9/recovered.edits/8.seqid, newMaxSeqId=8, maxSeqId=1 2024-11-26T14:31:25,019 INFO [RS_CLOSE_REGION-regionserver/19945b63c2b4:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1973): Closed testReplayEditsAfterRegionMovedWithMultiCF,,1732631484097.40329a66ba4e990f8463ecffc5dfdaa9. 2024-11-26T14:31:25,020 DEBUG [RS_CLOSE_REGION-regionserver/19945b63c2b4:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1676): Region close journal for 40329a66ba4e990f8463ecffc5dfdaa9: Waiting for close lock at 1732631484970Running coprocessor pre-close hooks at 1732631484970Disabling compacts and flushes for region at 1732631484970Disabling writes for close at 1732631484970Obtaining lock to block concurrent updates at 1732631484971 (+1 ms)Preparing flush snapshotting stores in 40329a66ba4e990f8463ecffc5dfdaa9 at 1732631484971Finished memstore snapshotting testReplayEditsAfterRegionMovedWithMultiCF,,1732631484097.40329a66ba4e990f8463ecffc5dfdaa9., syncing WAL and waiting on mvcc, flushsize=dataSize=31, getHeapSize=584, getOffHeapSize=0, getCellsCount=1 at 1732631484971Flushing stores of testReplayEditsAfterRegionMovedWithMultiCF,,1732631484097.40329a66ba4e990f8463ecffc5dfdaa9. at 1732631484972 (+1 ms)Flushing 40329a66ba4e990f8463ecffc5dfdaa9/cf1: creating writer at 1732631484972Flushing 40329a66ba4e990f8463ecffc5dfdaa9/cf1: appending metadata at 1732631484989 (+17 ms)Flushing 40329a66ba4e990f8463ecffc5dfdaa9/cf1: closing flushed file at 1732631484989Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@360149b: reopening flushed file at 1732631485003 (+14 ms)Finished flush of dataSize ~31 B/31, heapSize ~344 B/344, currentSize=0 B/0 for 40329a66ba4e990f8463ecffc5dfdaa9 in 42ms, sequenceid=5, compaction requested=false at 1732631485012 (+9 ms)Writing region close event to WAL at 1732631485013 (+1 ms)Running coprocessor post-close hooks at 1732631485018 (+5 ms)Closed at 1732631485019 (+1 ms) 2024-11-26T14:31:25,020 INFO [RS_CLOSE_REGION-regionserver/19945b63c2b4:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegionServer(3302): Adding 40329a66ba4e990f8463ecffc5dfdaa9 move to 19945b63c2b4,40179,1732631469496 record at close sequenceid=5 2024-11-26T14:31:25,023 INFO [RS_CLOSE_REGION-regionserver/19945b63c2b4:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] handler.UnassignRegionHandler(157): Closed 40329a66ba4e990f8463ecffc5dfdaa9 2024-11-26T14:31:25,024 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=40329a66ba4e990f8463ecffc5dfdaa9, regionState=CLOSED 2024-11-26T14:31:25,026 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=8, ppid=7, state=RUNNABLE, hasLock=false; CloseRegionProcedure 40329a66ba4e990f8463ecffc5dfdaa9, server=19945b63c2b4,35419,1732631469384 because future has completed 2024-11-26T14:31:25,030 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=8, resume processing ppid=7 2024-11-26T14:31:25,030 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=8, ppid=7, state=SUCCESS, hasLock=false; CloseRegionProcedure 40329a66ba4e990f8463ecffc5dfdaa9, server=19945b63c2b4,35419,1732631469384 in 219 msec 2024-11-26T14:31:25,031 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(269): Starting pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=40329a66ba4e990f8463ecffc5dfdaa9, REOPEN/MOVE; state=CLOSED, location=19945b63c2b4,40179,1732631469496; forceNewPlan=false, retain=false 2024-11-26T14:31:25,181 INFO [19945b63c2b4:43577 {}] balancer.BaseLoadBalancer(388): Reassigned 1 regions. 1 retained the pre-restart assignment. 2024-11-26T14:31:25,182 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=40329a66ba4e990f8463ecffc5dfdaa9, regionState=OPENING, regionLocation=19945b63c2b4,40179,1732631469496 2024-11-26T14:31:25,187 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=40329a66ba4e990f8463ecffc5dfdaa9, REOPEN/MOVE because future has completed 2024-11-26T14:31:25,189 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=9, ppid=7, state=RUNNABLE, hasLock=false; OpenRegionProcedure 40329a66ba4e990f8463ecffc5dfdaa9, server=19945b63c2b4,40179,1732631469496}] 2024-11-26T14:31:25,352 INFO [RS_OPEN_REGION-regionserver/19945b63c2b4:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] handler.AssignRegionHandler(132): Open testReplayEditsAfterRegionMovedWithMultiCF,,1732631484097.40329a66ba4e990f8463ecffc5dfdaa9. 2024-11-26T14:31:25,352 DEBUG [RS_OPEN_REGION-regionserver/19945b63c2b4:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(7752): Opening region: {ENCODED => 40329a66ba4e990f8463ecffc5dfdaa9, NAME => 'testReplayEditsAfterRegionMovedWithMultiCF,,1732631484097.40329a66ba4e990f8463ecffc5dfdaa9.', STARTKEY => '', ENDKEY => ''} 2024-11-26T14:31:25,353 DEBUG [RS_OPEN_REGION-regionserver/19945b63c2b4:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testReplayEditsAfterRegionMovedWithMultiCF 40329a66ba4e990f8463ecffc5dfdaa9 2024-11-26T14:31:25,353 DEBUG [RS_OPEN_REGION-regionserver/19945b63c2b4:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(898): Instantiated testReplayEditsAfterRegionMovedWithMultiCF,,1732631484097.40329a66ba4e990f8463ecffc5dfdaa9.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-26T14:31:25,353 DEBUG [RS_OPEN_REGION-regionserver/19945b63c2b4:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(7794): checking encryption for 40329a66ba4e990f8463ecffc5dfdaa9 2024-11-26T14:31:25,353 DEBUG [RS_OPEN_REGION-regionserver/19945b63c2b4:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(7797): checking classloading for 40329a66ba4e990f8463ecffc5dfdaa9 2024-11-26T14:31:25,355 INFO [StoreOpener-40329a66ba4e990f8463ecffc5dfdaa9-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf1 of region 40329a66ba4e990f8463ecffc5dfdaa9 2024-11-26T14:31:25,356 INFO [StoreOpener-40329a66ba4e990f8463ecffc5dfdaa9-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 40329a66ba4e990f8463ecffc5dfdaa9 columnFamilyName cf1 2024-11-26T14:31:25,356 DEBUG [StoreOpener-40329a66ba4e990f8463ecffc5dfdaa9-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:31:25,364 DEBUG [StoreOpener-40329a66ba4e990f8463ecffc5dfdaa9-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:44999/user/jenkins/test-data/2de93217-0593-bca0-5c5f-879b6362c76f/data/default/testReplayEditsAfterRegionMovedWithMultiCF/40329a66ba4e990f8463ecffc5dfdaa9/cf1/6b2bb3a954f34f899f665e29662182d0 2024-11-26T14:31:25,364 INFO [StoreOpener-40329a66ba4e990f8463ecffc5dfdaa9-1 {}] regionserver.HStore(327): Store=40329a66ba4e990f8463ecffc5dfdaa9/cf1, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-26T14:31:25,365 INFO [StoreOpener-40329a66ba4e990f8463ecffc5dfdaa9-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf2 of region 40329a66ba4e990f8463ecffc5dfdaa9 2024-11-26T14:31:25,366 INFO [StoreOpener-40329a66ba4e990f8463ecffc5dfdaa9-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 40329a66ba4e990f8463ecffc5dfdaa9 columnFamilyName cf2 2024-11-26T14:31:25,366 DEBUG [StoreOpener-40329a66ba4e990f8463ecffc5dfdaa9-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:31:25,367 INFO [StoreOpener-40329a66ba4e990f8463ecffc5dfdaa9-1 {}] regionserver.HStore(327): Store=40329a66ba4e990f8463ecffc5dfdaa9/cf2, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-26T14:31:25,367 DEBUG [RS_OPEN_REGION-regionserver/19945b63c2b4:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1038): replaying wal for 40329a66ba4e990f8463ecffc5dfdaa9 2024-11-26T14:31:25,368 DEBUG [RS_OPEN_REGION-regionserver/19945b63c2b4:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44999/user/jenkins/test-data/2de93217-0593-bca0-5c5f-879b6362c76f/data/default/testReplayEditsAfterRegionMovedWithMultiCF/40329a66ba4e990f8463ecffc5dfdaa9 2024-11-26T14:31:25,370 DEBUG [RS_OPEN_REGION-regionserver/19945b63c2b4:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44999/user/jenkins/test-data/2de93217-0593-bca0-5c5f-879b6362c76f/data/default/testReplayEditsAfterRegionMovedWithMultiCF/40329a66ba4e990f8463ecffc5dfdaa9 2024-11-26T14:31:25,371 DEBUG [RS_OPEN_REGION-regionserver/19945b63c2b4:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1048): stopping wal replay for 40329a66ba4e990f8463ecffc5dfdaa9 2024-11-26T14:31:25,371 DEBUG [RS_OPEN_REGION-regionserver/19945b63c2b4:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1060): Cleaning up temporary data for 40329a66ba4e990f8463ecffc5dfdaa9 2024-11-26T14:31:25,372 DEBUG [RS_OPEN_REGION-regionserver/19945b63c2b4:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsAfterRegionMovedWithMultiCF descriptor;using region.getMemStoreFlushHeapSize/# of families (64.0 M)) instead. 2024-11-26T14:31:25,374 DEBUG [RS_OPEN_REGION-regionserver/19945b63c2b4:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1093): writing seq id for 40329a66ba4e990f8463ecffc5dfdaa9 2024-11-26T14:31:25,375 INFO [RS_OPEN_REGION-regionserver/19945b63c2b4:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1114): Opened 40329a66ba4e990f8463ecffc5dfdaa9; next sequenceid=9; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=67707912, jitterRate=0.00892651081085205}}}, FlushLargeStoresPolicy{flushSizeLowerBound=67108864} 2024-11-26T14:31:25,376 DEBUG [RS_OPEN_REGION-regionserver/19945b63c2b4:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 40329a66ba4e990f8463ecffc5dfdaa9 2024-11-26T14:31:25,377 DEBUG [RS_OPEN_REGION-regionserver/19945b63c2b4:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1006): Region open journal for 40329a66ba4e990f8463ecffc5dfdaa9: Running coprocessor pre-open hook at 1732631485353Writing region info on filesystem at 1732631485353Initializing all the Stores at 1732631485354 (+1 ms)Instantiating store for column family {NAME => 'cf1', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732631485354Instantiating store for column family {NAME => 'cf2', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732631485355 (+1 ms)Cleaning up temporary data from old regions at 1732631485371 (+16 ms)Running coprocessor post-open hooks at 1732631485376 (+5 ms)Region opened successfully at 1732631485377 (+1 ms) 2024-11-26T14:31:25,378 INFO [RS_OPEN_REGION-regionserver/19945b63c2b4:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegionServer(2236): Post open deploy tasks for testReplayEditsAfterRegionMovedWithMultiCF,,1732631484097.40329a66ba4e990f8463ecffc5dfdaa9., pid=9, masterSystemTime=1732631485343 2024-11-26T14:31:25,381 DEBUG [RS_OPEN_REGION-regionserver/19945b63c2b4:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegionServer(2266): Finished post open deploy task for testReplayEditsAfterRegionMovedWithMultiCF,,1732631484097.40329a66ba4e990f8463ecffc5dfdaa9. 2024-11-26T14:31:25,381 INFO [RS_OPEN_REGION-regionserver/19945b63c2b4:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] handler.AssignRegionHandler(153): Opened testReplayEditsAfterRegionMovedWithMultiCF,,1732631484097.40329a66ba4e990f8463ecffc5dfdaa9. 2024-11-26T14:31:25,382 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=40329a66ba4e990f8463ecffc5dfdaa9, regionState=OPEN, openSeqNum=9, regionLocation=19945b63c2b4,40179,1732631469496 2024-11-26T14:31:25,386 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=9, ppid=7, state=RUNNABLE, hasLock=false; OpenRegionProcedure 40329a66ba4e990f8463ecffc5dfdaa9, server=19945b63c2b4,40179,1732631469496 because future has completed 2024-11-26T14:31:25,391 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=9, resume processing ppid=7 2024-11-26T14:31:25,391 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=9, ppid=7, state=SUCCESS, hasLock=false; OpenRegionProcedure 40329a66ba4e990f8463ecffc5dfdaa9, server=19945b63c2b4,40179,1732631469496 in 199 msec 2024-11-26T14:31:25,393 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=7, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=40329a66ba4e990f8463ecffc5dfdaa9, REOPEN/MOVE in 591 msec 2024-11-26T14:31:25,409 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-26T14:31:25,411 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-4-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:34816, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-26T14:31:25,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35419 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 139 connection: 172.17.0.3:47654 deadline: 1732631545416, exception=org.apache.hadoop.hbase.exceptions.RegionMovedException: Region moved to: hostname=19945b63c2b4 port=40179 startCode=1732631469496. As of locationSeqNum=5. 2024-11-26T14:31:25,423 DEBUG [RPCClient-NioEventLoopGroup-6-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1732631484097.40329a66ba4e990f8463ecffc5dfdaa9., hostname=19945b63c2b4,35419,1732631469384, seqNum=2 , the old value is region=testReplayEditsAfterRegionMovedWithMultiCF,,1732631484097.40329a66ba4e990f8463ecffc5dfdaa9., hostname=19945b63c2b4,35419,1732631469384, seqNum=2, error=org.apache.hadoop.hbase.exceptions.RegionMovedException: Region moved to: hostname=19945b63c2b4 port=40179 startCode=1732631469496. As of locationSeqNum=5. 2024-11-26T14:31:25,423 DEBUG [RPCClient-NioEventLoopGroup-6-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1732631484097.40329a66ba4e990f8463ecffc5dfdaa9., hostname=19945b63c2b4,35419,1732631469384, seqNum=2 is org.apache.hadoop.hbase.exceptions.RegionMovedException: Region moved to: hostname=19945b63c2b4 port=40179 startCode=1732631469496. As of locationSeqNum=5. 2024-11-26T14:31:25,423 DEBUG [RPCClient-NioEventLoopGroup-6-5 {}] client.AsyncRegionLocatorHelper(84): Try updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1732631484097.40329a66ba4e990f8463ecffc5dfdaa9., hostname=19945b63c2b4,35419,1732631469384, seqNum=2 with the new location region=testReplayEditsAfterRegionMovedWithMultiCF,,1732631484097.40329a66ba4e990f8463ecffc5dfdaa9., hostname=19945b63c2b4,40179,1732631469496, seqNum=5 constructed by org.apache.hadoop.hbase.exceptions.RegionMovedException: Region moved to: hostname=19945b63c2b4 port=40179 startCode=1732631469496. As of locationSeqNum=5. 2024-11-26T14:31:25,536 DEBUG [Async-Client-Retry-Timer-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-26T14:31:25,538 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-4-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:34830, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-26T14:31:25,547 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 40329a66ba4e990f8463ecffc5dfdaa9 2/2 column families, dataSize=50 B heapSize=720 B 2024-11-26T14:31:25,570 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44999/user/jenkins/test-data/2de93217-0593-bca0-5c5f-879b6362c76f/data/default/testReplayEditsAfterRegionMovedWithMultiCF/40329a66ba4e990f8463ecffc5dfdaa9/.tmp/cf1/752258afe2fa4a7faeb4f1788ea3f2b7 is 29, key is r1/cf1:/1732631485539/DeleteFamily/seqid=0 2024-11-26T14:31:25,577 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36277 is added to blk_1073741888_1066 (size=4906) 2024-11-26T14:31:25,577 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38147 is added to blk_1073741888_1066 (size=4906) 2024-11-26T14:31:25,577 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36307 is added to blk_1073741888_1066 (size=4906) 2024-11-26T14:31:25,577 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=25 B at sequenceid=12 (bloomFilter=false), to=hdfs://localhost:44999/user/jenkins/test-data/2de93217-0593-bca0-5c5f-879b6362c76f/data/default/testReplayEditsAfterRegionMovedWithMultiCF/40329a66ba4e990f8463ecffc5dfdaa9/.tmp/cf1/752258afe2fa4a7faeb4f1788ea3f2b7 2024-11-26T14:31:25,585 INFO [Time-limited test {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 752258afe2fa4a7faeb4f1788ea3f2b7 2024-11-26T14:31:25,601 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44999/user/jenkins/test-data/2de93217-0593-bca0-5c5f-879b6362c76f/data/default/testReplayEditsAfterRegionMovedWithMultiCF/40329a66ba4e990f8463ecffc5dfdaa9/.tmp/cf2/7a1cc1c91023406b97c162e37a7be23a is 29, key is r1/cf2:/1732631485539/DeleteFamily/seqid=0 2024-11-26T14:31:25,608 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36277 is added to blk_1073741889_1067 (size=4906) 2024-11-26T14:31:25,608 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36307 is added to blk_1073741889_1067 (size=4906) 2024-11-26T14:31:25,608 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38147 is added to blk_1073741889_1067 (size=4906) 2024-11-26T14:31:25,608 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=25 B at sequenceid=12 (bloomFilter=false), to=hdfs://localhost:44999/user/jenkins/test-data/2de93217-0593-bca0-5c5f-879b6362c76f/data/default/testReplayEditsAfterRegionMovedWithMultiCF/40329a66ba4e990f8463ecffc5dfdaa9/.tmp/cf2/7a1cc1c91023406b97c162e37a7be23a 2024-11-26T14:31:25,614 INFO [Time-limited test {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 7a1cc1c91023406b97c162e37a7be23a 2024-11-26T14:31:25,615 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44999/user/jenkins/test-data/2de93217-0593-bca0-5c5f-879b6362c76f/data/default/testReplayEditsAfterRegionMovedWithMultiCF/40329a66ba4e990f8463ecffc5dfdaa9/.tmp/cf1/752258afe2fa4a7faeb4f1788ea3f2b7 as hdfs://localhost:44999/user/jenkins/test-data/2de93217-0593-bca0-5c5f-879b6362c76f/data/default/testReplayEditsAfterRegionMovedWithMultiCF/40329a66ba4e990f8463ecffc5dfdaa9/cf1/752258afe2fa4a7faeb4f1788ea3f2b7 2024-11-26T14:31:25,623 INFO [Time-limited test {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 752258afe2fa4a7faeb4f1788ea3f2b7 2024-11-26T14:31:25,623 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44999/user/jenkins/test-data/2de93217-0593-bca0-5c5f-879b6362c76f/data/default/testReplayEditsAfterRegionMovedWithMultiCF/40329a66ba4e990f8463ecffc5dfdaa9/cf1/752258afe2fa4a7faeb4f1788ea3f2b7, entries=1, sequenceid=12, filesize=4.8 K 2024-11-26T14:31:25,624 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44999/user/jenkins/test-data/2de93217-0593-bca0-5c5f-879b6362c76f/data/default/testReplayEditsAfterRegionMovedWithMultiCF/40329a66ba4e990f8463ecffc5dfdaa9/.tmp/cf2/7a1cc1c91023406b97c162e37a7be23a as hdfs://localhost:44999/user/jenkins/test-data/2de93217-0593-bca0-5c5f-879b6362c76f/data/default/testReplayEditsAfterRegionMovedWithMultiCF/40329a66ba4e990f8463ecffc5dfdaa9/cf2/7a1cc1c91023406b97c162e37a7be23a 2024-11-26T14:31:25,632 INFO [Time-limited test {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 7a1cc1c91023406b97c162e37a7be23a 2024-11-26T14:31:25,632 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44999/user/jenkins/test-data/2de93217-0593-bca0-5c5f-879b6362c76f/data/default/testReplayEditsAfterRegionMovedWithMultiCF/40329a66ba4e990f8463ecffc5dfdaa9/cf2/7a1cc1c91023406b97c162e37a7be23a, entries=1, sequenceid=12, filesize=4.8 K 2024-11-26T14:31:25,633 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~50 B/50, heapSize ~688 B/688, currentSize=0 B/0 for 40329a66ba4e990f8463ecffc5dfdaa9 in 86ms, sequenceid=12, compaction requested=false 2024-11-26T14:31:25,633 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for 40329a66ba4e990f8463ecffc5dfdaa9: 2024-11-26T14:31:25,636 DEBUG [Time-limited test {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 2 store files, 0 compacting, 2 eligible, 16 blocking 2024-11-26T14:31:25,637 DEBUG [Time-limited test {}] regionserver.HStore(1541): 40329a66ba4e990f8463ecffc5dfdaa9/cf1 is initiating major compaction (all files) 2024-11-26T14:31:25,637 INFO [Time-limited test {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-26T14:31:25,637 INFO [Time-limited test {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-26T14:31:25,638 INFO [Time-limited test {}] regionserver.HRegion(2416): Starting compaction of 40329a66ba4e990f8463ecffc5dfdaa9/cf1 in testReplayEditsAfterRegionMovedWithMultiCF,,1732631484097.40329a66ba4e990f8463ecffc5dfdaa9. 2024-11-26T14:31:25,638 INFO [Time-limited test {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44999/user/jenkins/test-data/2de93217-0593-bca0-5c5f-879b6362c76f/data/default/testReplayEditsAfterRegionMovedWithMultiCF/40329a66ba4e990f8463ecffc5dfdaa9/cf1/6b2bb3a954f34f899f665e29662182d0, hdfs://localhost:44999/user/jenkins/test-data/2de93217-0593-bca0-5c5f-879b6362c76f/data/default/testReplayEditsAfterRegionMovedWithMultiCF/40329a66ba4e990f8463ecffc5dfdaa9/cf1/752258afe2fa4a7faeb4f1788ea3f2b7] into tmpdir=hdfs://localhost:44999/user/jenkins/test-data/2de93217-0593-bca0-5c5f-879b6362c76f/data/default/testReplayEditsAfterRegionMovedWithMultiCF/40329a66ba4e990f8463ecffc5dfdaa9/.tmp, totalSize=9.5 K 2024-11-26T14:31:25,639 DEBUG [Time-limited test {}] compactions.Compactor(225): Compacting 6b2bb3a954f34f899f665e29662182d0, keycount=1, bloomtype=NONE, size=4.7 K, encoding=NONE, compression=NONE, seqNum=5, earliestPutTs=1732631484781 2024-11-26T14:31:25,640 DEBUG [Time-limited test {}] compactions.Compactor(225): Compacting 752258afe2fa4a7faeb4f1788ea3f2b7, keycount=1, bloomtype=NONE, size=4.8 K, encoding=NONE, compression=NONE, seqNum=12, earliestPutTs=9223372036854775807 2024-11-26T14:31:25,652 INFO [Time-limited test {}] throttle.PressureAwareThroughputController(145): 40329a66ba4e990f8463ecffc5dfdaa9#cf1#compaction#16 average throughput is NaN MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-26T14:31:25,658 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36277 is added to blk_1073741890_1068 (size=4626) 2024-11-26T14:31:25,658 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38147 is added to blk_1073741890_1068 (size=4626) 2024-11-26T14:31:25,658 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36307 is added to blk_1073741890_1068 (size=4626) 2024-11-26T14:31:25,666 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44999/user/jenkins/test-data/2de93217-0593-bca0-5c5f-879b6362c76f/data/default/testReplayEditsAfterRegionMovedWithMultiCF/40329a66ba4e990f8463ecffc5dfdaa9/.tmp/cf1/996eb589593143149c38b54e30a0fb3f as hdfs://localhost:44999/user/jenkins/test-data/2de93217-0593-bca0-5c5f-879b6362c76f/data/default/testReplayEditsAfterRegionMovedWithMultiCF/40329a66ba4e990f8463ecffc5dfdaa9/cf1/996eb589593143149c38b54e30a0fb3f 2024-11-26T14:31:25,680 INFO [Time-limited test {}] regionserver.HStore(1337): Completed major compaction of 2 (all) file(s) in 40329a66ba4e990f8463ecffc5dfdaa9/cf1 of 40329a66ba4e990f8463ecffc5dfdaa9 into 996eb589593143149c38b54e30a0fb3f(size=4.5 K), total size for store is 4.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T14:31:25,680 DEBUG [Time-limited test {}] regionserver.HRegion(2446): Compaction status journal for 40329a66ba4e990f8463ecffc5dfdaa9: 2024-11-26T14:31:25,681 DEBUG [Time-limited test {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 1 store files, 0 compacting, 1 eligible, 16 blocking 2024-11-26T14:31:25,681 DEBUG [Time-limited test {}] regionserver.HStore(1541): 40329a66ba4e990f8463ecffc5dfdaa9/cf2 is initiating major compaction (all files) 2024-11-26T14:31:25,681 INFO [Time-limited test {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-26T14:31:25,681 INFO [Time-limited test {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-26T14:31:25,681 INFO [Time-limited test {}] regionserver.HRegion(2416): Starting compaction of 40329a66ba4e990f8463ecffc5dfdaa9/cf2 in testReplayEditsAfterRegionMovedWithMultiCF,,1732631484097.40329a66ba4e990f8463ecffc5dfdaa9. 2024-11-26T14:31:25,681 INFO [Time-limited test {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44999/user/jenkins/test-data/2de93217-0593-bca0-5c5f-879b6362c76f/data/default/testReplayEditsAfterRegionMovedWithMultiCF/40329a66ba4e990f8463ecffc5dfdaa9/cf2/7a1cc1c91023406b97c162e37a7be23a] into tmpdir=hdfs://localhost:44999/user/jenkins/test-data/2de93217-0593-bca0-5c5f-879b6362c76f/data/default/testReplayEditsAfterRegionMovedWithMultiCF/40329a66ba4e990f8463ecffc5dfdaa9/.tmp, totalSize=4.8 K 2024-11-26T14:31:25,682 DEBUG [Time-limited test {}] compactions.Compactor(225): Compacting 7a1cc1c91023406b97c162e37a7be23a, keycount=1, bloomtype=NONE, size=4.8 K, encoding=NONE, compression=NONE, seqNum=12, earliestPutTs=9223372036854775807 2024-11-26T14:31:25,688 INFO [Time-limited test {}] throttle.PressureAwareThroughputController(145): 40329a66ba4e990f8463ecffc5dfdaa9#cf2#compaction#17 average throughput is NaN MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-26T14:31:25,695 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36277 is added to blk_1073741891_1069 (size=4592) 2024-11-26T14:31:25,695 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38147 is added to blk_1073741891_1069 (size=4592) 2024-11-26T14:31:25,695 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36307 is added to blk_1073741891_1069 (size=4592) 2024-11-26T14:31:25,703 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44999/user/jenkins/test-data/2de93217-0593-bca0-5c5f-879b6362c76f/data/default/testReplayEditsAfterRegionMovedWithMultiCF/40329a66ba4e990f8463ecffc5dfdaa9/.tmp/cf2/aad6938c9a964b5d92fe3112370fd466 as hdfs://localhost:44999/user/jenkins/test-data/2de93217-0593-bca0-5c5f-879b6362c76f/data/default/testReplayEditsAfterRegionMovedWithMultiCF/40329a66ba4e990f8463ecffc5dfdaa9/cf2/aad6938c9a964b5d92fe3112370fd466 2024-11-26T14:31:25,711 INFO [Time-limited test {}] regionserver.HStore(1337): Completed major compaction of 1 (all) file(s) in 40329a66ba4e990f8463ecffc5dfdaa9/cf2 of 40329a66ba4e990f8463ecffc5dfdaa9 into aad6938c9a964b5d92fe3112370fd466(size=4.5 K), total size for store is 4.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T14:31:25,711 DEBUG [Time-limited test {}] regionserver.HRegion(2446): Compaction status journal for 40329a66ba4e990f8463ecffc5dfdaa9: 2024-11-26T14:31:25,716 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43577 {}] master.HMaster(2410): Client=jenkins//172.17.0.3 move hri=40329a66ba4e990f8463ecffc5dfdaa9, source=19945b63c2b4,40179,1732631469496, destination=19945b63c2b4,35419,1732631469384, warming up region on 19945b63c2b4,35419,1732631469384 2024-11-26T14:31:25,716 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43577 {}] master.HMaster(2414): Client=jenkins//172.17.0.3 move hri=40329a66ba4e990f8463ecffc5dfdaa9, source=19945b63c2b4,40179,1732631469496, destination=19945b63c2b4,35419,1732631469384, running balancer 2024-11-26T14:31:25,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43577 {}] procedure2.ProcedureExecutor(1139): Stored pid=10, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=40329a66ba4e990f8463ecffc5dfdaa9, REOPEN/MOVE 2024-11-26T14:31:25,718 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=10, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=40329a66ba4e990f8463ecffc5dfdaa9, REOPEN/MOVE 2024-11-26T14:31:25,720 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=10 updating hbase:meta row=40329a66ba4e990f8463ecffc5dfdaa9, regionState=CLOSING, regionLocation=19945b63c2b4,40179,1732631469496 2024-11-26T14:31:25,721 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35419 {}] regionserver.RSRpcServices(2066): Warmup testReplayEditsAfterRegionMovedWithMultiCF,,1732631484097.40329a66ba4e990f8463ecffc5dfdaa9. 2024-11-26T14:31:25,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35419 {}] regionserver.HRegion(7855): Warmup {ENCODED => 40329a66ba4e990f8463ecffc5dfdaa9, NAME => 'testReplayEditsAfterRegionMovedWithMultiCF,,1732631484097.40329a66ba4e990f8463ecffc5dfdaa9.', STARTKEY => '', ENDKEY => ''} 2024-11-26T14:31:25,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35419 {}] regionserver.HRegion(898): Instantiated testReplayEditsAfterRegionMovedWithMultiCF,,1732631484097.40329a66ba4e990f8463ecffc5dfdaa9.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-26T14:31:25,723 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=10, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=40329a66ba4e990f8463ecffc5dfdaa9, REOPEN/MOVE because future has completed 2024-11-26T14:31:25,724 DEBUG [PEWorker-5 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-26T14:31:25,724 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=11, ppid=10, state=RUNNABLE, hasLock=false; CloseRegionProcedure 40329a66ba4e990f8463ecffc5dfdaa9, server=19945b63c2b4,40179,1732631469496}] 2024-11-26T14:31:25,725 INFO [StoreOpener-40329a66ba4e990f8463ecffc5dfdaa9-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf1 of region 40329a66ba4e990f8463ecffc5dfdaa9 2024-11-26T14:31:25,726 INFO [StoreOpener-40329a66ba4e990f8463ecffc5dfdaa9-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 40329a66ba4e990f8463ecffc5dfdaa9 columnFamilyName cf1 2024-11-26T14:31:25,726 DEBUG [StoreOpener-40329a66ba4e990f8463ecffc5dfdaa9-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:31:25,734 DEBUG [StoreOpener-40329a66ba4e990f8463ecffc5dfdaa9-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:44999/user/jenkins/test-data/2de93217-0593-bca0-5c5f-879b6362c76f/data/default/testReplayEditsAfterRegionMovedWithMultiCF/40329a66ba4e990f8463ecffc5dfdaa9/cf1/6b2bb3a954f34f899f665e29662182d0 2024-11-26T14:31:25,740 INFO [StoreFileOpener-40329a66ba4e990f8463ecffc5dfdaa9-cf1-1 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 752258afe2fa4a7faeb4f1788ea3f2b7 2024-11-26T14:31:25,740 DEBUG [StoreOpener-40329a66ba4e990f8463ecffc5dfdaa9-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:44999/user/jenkins/test-data/2de93217-0593-bca0-5c5f-879b6362c76f/data/default/testReplayEditsAfterRegionMovedWithMultiCF/40329a66ba4e990f8463ecffc5dfdaa9/cf1/752258afe2fa4a7faeb4f1788ea3f2b7 2024-11-26T14:31:25,747 DEBUG [StoreOpener-40329a66ba4e990f8463ecffc5dfdaa9-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:44999/user/jenkins/test-data/2de93217-0593-bca0-5c5f-879b6362c76f/data/default/testReplayEditsAfterRegionMovedWithMultiCF/40329a66ba4e990f8463ecffc5dfdaa9/cf1/996eb589593143149c38b54e30a0fb3f 2024-11-26T14:31:25,747 INFO [StoreOpener-40329a66ba4e990f8463ecffc5dfdaa9-1 {}] regionserver.HStore(327): Store=40329a66ba4e990f8463ecffc5dfdaa9/cf1, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-26T14:31:25,748 INFO [StoreOpener-40329a66ba4e990f8463ecffc5dfdaa9-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf2 of region 40329a66ba4e990f8463ecffc5dfdaa9 2024-11-26T14:31:25,749 INFO [StoreOpener-40329a66ba4e990f8463ecffc5dfdaa9-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 40329a66ba4e990f8463ecffc5dfdaa9 columnFamilyName cf2 2024-11-26T14:31:25,749 DEBUG [StoreOpener-40329a66ba4e990f8463ecffc5dfdaa9-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:31:25,757 INFO [StoreFileOpener-40329a66ba4e990f8463ecffc5dfdaa9-cf2-1 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 7a1cc1c91023406b97c162e37a7be23a 2024-11-26T14:31:25,757 DEBUG [StoreOpener-40329a66ba4e990f8463ecffc5dfdaa9-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:44999/user/jenkins/test-data/2de93217-0593-bca0-5c5f-879b6362c76f/data/default/testReplayEditsAfterRegionMovedWithMultiCF/40329a66ba4e990f8463ecffc5dfdaa9/cf2/7a1cc1c91023406b97c162e37a7be23a 2024-11-26T14:31:25,763 DEBUG [StoreOpener-40329a66ba4e990f8463ecffc5dfdaa9-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:44999/user/jenkins/test-data/2de93217-0593-bca0-5c5f-879b6362c76f/data/default/testReplayEditsAfterRegionMovedWithMultiCF/40329a66ba4e990f8463ecffc5dfdaa9/cf2/aad6938c9a964b5d92fe3112370fd466 2024-11-26T14:31:25,763 INFO [StoreOpener-40329a66ba4e990f8463ecffc5dfdaa9-1 {}] regionserver.HStore(327): Store=40329a66ba4e990f8463ecffc5dfdaa9/cf2, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-26T14:31:25,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35419 {}] regionserver.HRegion(1722): Closing 40329a66ba4e990f8463ecffc5dfdaa9, disabling compactions & flushes 2024-11-26T14:31:25,764 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35419 {}] regionserver.HRegion(1755): Closing region testReplayEditsAfterRegionMovedWithMultiCF,,1732631484097.40329a66ba4e990f8463ecffc5dfdaa9. 2024-11-26T14:31:25,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35419 {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1732631484097.40329a66ba4e990f8463ecffc5dfdaa9. 2024-11-26T14:31:25,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35419 {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1732631484097.40329a66ba4e990f8463ecffc5dfdaa9. after waiting 0 ms 2024-11-26T14:31:25,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35419 {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsAfterRegionMovedWithMultiCF,,1732631484097.40329a66ba4e990f8463ecffc5dfdaa9. 2024-11-26T14:31:25,765 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35419 {}] regionserver.HRegion(1973): Closed testReplayEditsAfterRegionMovedWithMultiCF,,1732631484097.40329a66ba4e990f8463ecffc5dfdaa9. 2024-11-26T14:31:25,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35419 {}] regionserver.HRegion(1676): Region close journal for 40329a66ba4e990f8463ecffc5dfdaa9: Waiting for close lock at 1732631485764Disabling compacts and flushes for region at 1732631485764Disabling writes for close at 1732631485764Writing region close event to WAL at 1732631485765 (+1 ms)Closed at 1732631485765 2024-11-26T14:31:25,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43577 {}] procedure.ProcedureSyncWait(219): waitFor pid=10 2024-11-26T14:31:25,879 INFO [RS_CLOSE_REGION-regionserver/19945b63c2b4:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] handler.UnassignRegionHandler(122): Close 40329a66ba4e990f8463ecffc5dfdaa9 2024-11-26T14:31:25,879 DEBUG [RS_CLOSE_REGION-regionserver/19945b63c2b4:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-26T14:31:25,879 DEBUG [RS_CLOSE_REGION-regionserver/19945b63c2b4:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] regionserver.HRegion(1722): Closing 40329a66ba4e990f8463ecffc5dfdaa9, disabling compactions & flushes 2024-11-26T14:31:25,879 INFO [RS_CLOSE_REGION-regionserver/19945b63c2b4:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] regionserver.HRegion(1755): Closing region testReplayEditsAfterRegionMovedWithMultiCF,,1732631484097.40329a66ba4e990f8463ecffc5dfdaa9. 2024-11-26T14:31:25,879 DEBUG [RS_CLOSE_REGION-regionserver/19945b63c2b4:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1732631484097.40329a66ba4e990f8463ecffc5dfdaa9. 2024-11-26T14:31:25,879 DEBUG [RS_CLOSE_REGION-regionserver/19945b63c2b4:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1732631484097.40329a66ba4e990f8463ecffc5dfdaa9. after waiting 0 ms 2024-11-26T14:31:25,879 DEBUG [RS_CLOSE_REGION-regionserver/19945b63c2b4:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsAfterRegionMovedWithMultiCF,,1732631484097.40329a66ba4e990f8463ecffc5dfdaa9. 2024-11-26T14:31:25,880 DEBUG [StoreCloser-testReplayEditsAfterRegionMovedWithMultiCF,,1732631484097.40329a66ba4e990f8463ecffc5dfdaa9.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:44999/user/jenkins/test-data/2de93217-0593-bca0-5c5f-879b6362c76f/data/default/testReplayEditsAfterRegionMovedWithMultiCF/40329a66ba4e990f8463ecffc5dfdaa9/cf1/6b2bb3a954f34f899f665e29662182d0, hdfs://localhost:44999/user/jenkins/test-data/2de93217-0593-bca0-5c5f-879b6362c76f/data/default/testReplayEditsAfterRegionMovedWithMultiCF/40329a66ba4e990f8463ecffc5dfdaa9/cf1/752258afe2fa4a7faeb4f1788ea3f2b7] to archive 2024-11-26T14:31:25,883 DEBUG [StoreCloser-testReplayEditsAfterRegionMovedWithMultiCF,,1732631484097.40329a66ba4e990f8463ecffc5dfdaa9.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-26T14:31:25,887 DEBUG [StoreCloser-testReplayEditsAfterRegionMovedWithMultiCF,,1732631484097.40329a66ba4e990f8463ecffc5dfdaa9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44999/user/jenkins/test-data/2de93217-0593-bca0-5c5f-879b6362c76f/data/default/testReplayEditsAfterRegionMovedWithMultiCF/40329a66ba4e990f8463ecffc5dfdaa9/cf1/6b2bb3a954f34f899f665e29662182d0 to hdfs://localhost:44999/user/jenkins/test-data/2de93217-0593-bca0-5c5f-879b6362c76f/archive/data/default/testReplayEditsAfterRegionMovedWithMultiCF/40329a66ba4e990f8463ecffc5dfdaa9/cf1/6b2bb3a954f34f899f665e29662182d0 2024-11-26T14:31:25,889 DEBUG [StoreCloser-testReplayEditsAfterRegionMovedWithMultiCF,,1732631484097.40329a66ba4e990f8463ecffc5dfdaa9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44999/user/jenkins/test-data/2de93217-0593-bca0-5c5f-879b6362c76f/data/default/testReplayEditsAfterRegionMovedWithMultiCF/40329a66ba4e990f8463ecffc5dfdaa9/cf1/752258afe2fa4a7faeb4f1788ea3f2b7 to hdfs://localhost:44999/user/jenkins/test-data/2de93217-0593-bca0-5c5f-879b6362c76f/archive/data/default/testReplayEditsAfterRegionMovedWithMultiCF/40329a66ba4e990f8463ecffc5dfdaa9/cf1/752258afe2fa4a7faeb4f1788ea3f2b7 2024-11-26T14:31:25,904 DEBUG [StoreCloser-testReplayEditsAfterRegionMovedWithMultiCF,,1732631484097.40329a66ba4e990f8463ecffc5dfdaa9.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:44999/user/jenkins/test-data/2de93217-0593-bca0-5c5f-879b6362c76f/data/default/testReplayEditsAfterRegionMovedWithMultiCF/40329a66ba4e990f8463ecffc5dfdaa9/cf2/7a1cc1c91023406b97c162e37a7be23a] to archive 2024-11-26T14:31:25,905 DEBUG [StoreCloser-testReplayEditsAfterRegionMovedWithMultiCF,,1732631484097.40329a66ba4e990f8463ecffc5dfdaa9.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-26T14:31:25,907 DEBUG [StoreCloser-testReplayEditsAfterRegionMovedWithMultiCF,,1732631484097.40329a66ba4e990f8463ecffc5dfdaa9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44999/user/jenkins/test-data/2de93217-0593-bca0-5c5f-879b6362c76f/data/default/testReplayEditsAfterRegionMovedWithMultiCF/40329a66ba4e990f8463ecffc5dfdaa9/cf2/7a1cc1c91023406b97c162e37a7be23a to hdfs://localhost:44999/user/jenkins/test-data/2de93217-0593-bca0-5c5f-879b6362c76f/archive/data/default/testReplayEditsAfterRegionMovedWithMultiCF/40329a66ba4e990f8463ecffc5dfdaa9/cf2/7a1cc1c91023406b97c162e37a7be23a 2024-11-26T14:31:25,912 DEBUG [RS_CLOSE_REGION-regionserver/19945b63c2b4:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44999/user/jenkins/test-data/2de93217-0593-bca0-5c5f-879b6362c76f/data/default/testReplayEditsAfterRegionMovedWithMultiCF/40329a66ba4e990f8463ecffc5dfdaa9/recovered.edits/17.seqid, newMaxSeqId=17, maxSeqId=8 2024-11-26T14:31:25,913 INFO [RS_CLOSE_REGION-regionserver/19945b63c2b4:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] regionserver.HRegion(1973): Closed testReplayEditsAfterRegionMovedWithMultiCF,,1732631484097.40329a66ba4e990f8463ecffc5dfdaa9. 2024-11-26T14:31:25,913 DEBUG [RS_CLOSE_REGION-regionserver/19945b63c2b4:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] regionserver.HRegion(1676): Region close journal for 40329a66ba4e990f8463ecffc5dfdaa9: Waiting for close lock at 1732631485879Running coprocessor pre-close hooks at 1732631485879Disabling compacts and flushes for region at 1732631485879Disabling writes for close at 1732631485879Writing region close event to WAL at 1732631485908 (+29 ms)Running coprocessor post-close hooks at 1732631485913 (+5 ms)Closed at 1732631485913 2024-11-26T14:31:25,913 INFO [RS_CLOSE_REGION-regionserver/19945b63c2b4:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] regionserver.HRegionServer(3302): Adding 40329a66ba4e990f8463ecffc5dfdaa9 move to 19945b63c2b4,35419,1732631469384 record at close sequenceid=12 2024-11-26T14:31:25,916 INFO [RS_CLOSE_REGION-regionserver/19945b63c2b4:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] handler.UnassignRegionHandler(157): Closed 40329a66ba4e990f8463ecffc5dfdaa9 2024-11-26T14:31:25,916 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=10 updating hbase:meta row=40329a66ba4e990f8463ecffc5dfdaa9, regionState=CLOSED 2024-11-26T14:31:25,919 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=11, ppid=10, state=RUNNABLE, hasLock=false; CloseRegionProcedure 40329a66ba4e990f8463ecffc5dfdaa9, server=19945b63c2b4,40179,1732631469496 because future has completed 2024-11-26T14:31:25,922 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=11, resume processing ppid=10 2024-11-26T14:31:25,923 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=11, ppid=10, state=SUCCESS, hasLock=false; CloseRegionProcedure 40329a66ba4e990f8463ecffc5dfdaa9, server=19945b63c2b4,40179,1732631469496 in 196 msec 2024-11-26T14:31:25,923 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=10, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=40329a66ba4e990f8463ecffc5dfdaa9, REOPEN/MOVE; state=CLOSED, location=19945b63c2b4,35419,1732631469384; forceNewPlan=false, retain=false 2024-11-26T14:31:26,074 INFO [19945b63c2b4:43577 {}] balancer.BaseLoadBalancer(388): Reassigned 1 regions. 1 retained the pre-restart assignment. 2024-11-26T14:31:26,074 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=10 updating hbase:meta row=40329a66ba4e990f8463ecffc5dfdaa9, regionState=OPENING, regionLocation=19945b63c2b4,35419,1732631469384 2024-11-26T14:31:26,077 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=10, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=40329a66ba4e990f8463ecffc5dfdaa9, REOPEN/MOVE because future has completed 2024-11-26T14:31:26,078 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=12, ppid=10, state=RUNNABLE, hasLock=false; OpenRegionProcedure 40329a66ba4e990f8463ecffc5dfdaa9, server=19945b63c2b4,35419,1732631469384}] 2024-11-26T14:31:26,239 INFO [RS_OPEN_REGION-regionserver/19945b63c2b4:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] handler.AssignRegionHandler(132): Open testReplayEditsAfterRegionMovedWithMultiCF,,1732631484097.40329a66ba4e990f8463ecffc5dfdaa9. 2024-11-26T14:31:26,239 DEBUG [RS_OPEN_REGION-regionserver/19945b63c2b4:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(7752): Opening region: {ENCODED => 40329a66ba4e990f8463ecffc5dfdaa9, NAME => 'testReplayEditsAfterRegionMovedWithMultiCF,,1732631484097.40329a66ba4e990f8463ecffc5dfdaa9.', STARTKEY => '', ENDKEY => ''} 2024-11-26T14:31:26,240 DEBUG [RS_OPEN_REGION-regionserver/19945b63c2b4:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testReplayEditsAfterRegionMovedWithMultiCF 40329a66ba4e990f8463ecffc5dfdaa9 2024-11-26T14:31:26,240 DEBUG [RS_OPEN_REGION-regionserver/19945b63c2b4:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(898): Instantiated testReplayEditsAfterRegionMovedWithMultiCF,,1732631484097.40329a66ba4e990f8463ecffc5dfdaa9.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-26T14:31:26,240 DEBUG [RS_OPEN_REGION-regionserver/19945b63c2b4:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(7794): checking encryption for 40329a66ba4e990f8463ecffc5dfdaa9 2024-11-26T14:31:26,240 DEBUG [RS_OPEN_REGION-regionserver/19945b63c2b4:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(7797): checking classloading for 40329a66ba4e990f8463ecffc5dfdaa9 2024-11-26T14:31:26,243 INFO [StoreOpener-40329a66ba4e990f8463ecffc5dfdaa9-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf1 of region 40329a66ba4e990f8463ecffc5dfdaa9 2024-11-26T14:31:26,244 INFO [StoreOpener-40329a66ba4e990f8463ecffc5dfdaa9-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 40329a66ba4e990f8463ecffc5dfdaa9 columnFamilyName cf1 2024-11-26T14:31:26,244 DEBUG [StoreOpener-40329a66ba4e990f8463ecffc5dfdaa9-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:31:26,250 DEBUG [StoreOpener-40329a66ba4e990f8463ecffc5dfdaa9-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:44999/user/jenkins/test-data/2de93217-0593-bca0-5c5f-879b6362c76f/data/default/testReplayEditsAfterRegionMovedWithMultiCF/40329a66ba4e990f8463ecffc5dfdaa9/cf1/996eb589593143149c38b54e30a0fb3f 2024-11-26T14:31:26,250 INFO [StoreOpener-40329a66ba4e990f8463ecffc5dfdaa9-1 {}] regionserver.HStore(327): Store=40329a66ba4e990f8463ecffc5dfdaa9/cf1, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-26T14:31:26,250 INFO [StoreOpener-40329a66ba4e990f8463ecffc5dfdaa9-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf2 of region 40329a66ba4e990f8463ecffc5dfdaa9 2024-11-26T14:31:26,251 INFO [StoreOpener-40329a66ba4e990f8463ecffc5dfdaa9-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 40329a66ba4e990f8463ecffc5dfdaa9 columnFamilyName cf2 2024-11-26T14:31:26,251 DEBUG [StoreOpener-40329a66ba4e990f8463ecffc5dfdaa9-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:31:26,258 DEBUG [StoreOpener-40329a66ba4e990f8463ecffc5dfdaa9-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:44999/user/jenkins/test-data/2de93217-0593-bca0-5c5f-879b6362c76f/data/default/testReplayEditsAfterRegionMovedWithMultiCF/40329a66ba4e990f8463ecffc5dfdaa9/cf2/aad6938c9a964b5d92fe3112370fd466 2024-11-26T14:31:26,258 INFO [StoreOpener-40329a66ba4e990f8463ecffc5dfdaa9-1 {}] regionserver.HStore(327): Store=40329a66ba4e990f8463ecffc5dfdaa9/cf2, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-26T14:31:26,259 DEBUG [RS_OPEN_REGION-regionserver/19945b63c2b4:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1038): replaying wal for 40329a66ba4e990f8463ecffc5dfdaa9 2024-11-26T14:31:26,259 DEBUG [RS_OPEN_REGION-regionserver/19945b63c2b4:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44999/user/jenkins/test-data/2de93217-0593-bca0-5c5f-879b6362c76f/data/default/testReplayEditsAfterRegionMovedWithMultiCF/40329a66ba4e990f8463ecffc5dfdaa9 2024-11-26T14:31:26,261 DEBUG [RS_OPEN_REGION-regionserver/19945b63c2b4:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44999/user/jenkins/test-data/2de93217-0593-bca0-5c5f-879b6362c76f/data/default/testReplayEditsAfterRegionMovedWithMultiCF/40329a66ba4e990f8463ecffc5dfdaa9 2024-11-26T14:31:26,262 DEBUG [RS_OPEN_REGION-regionserver/19945b63c2b4:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1048): stopping wal replay for 40329a66ba4e990f8463ecffc5dfdaa9 2024-11-26T14:31:26,262 DEBUG [RS_OPEN_REGION-regionserver/19945b63c2b4:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1060): Cleaning up temporary data for 40329a66ba4e990f8463ecffc5dfdaa9 2024-11-26T14:31:26,263 DEBUG [RS_OPEN_REGION-regionserver/19945b63c2b4:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsAfterRegionMovedWithMultiCF descriptor;using region.getMemStoreFlushHeapSize/# of families (64.0 M)) instead. 2024-11-26T14:31:26,264 DEBUG [RS_OPEN_REGION-regionserver/19945b63c2b4:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1093): writing seq id for 40329a66ba4e990f8463ecffc5dfdaa9 2024-11-26T14:31:26,265 INFO [RS_OPEN_REGION-regionserver/19945b63c2b4:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1114): Opened 40329a66ba4e990f8463ecffc5dfdaa9; next sequenceid=18; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=75416084, jitterRate=0.12378722429275513}}}, FlushLargeStoresPolicy{flushSizeLowerBound=67108864} 2024-11-26T14:31:26,265 DEBUG [RS_OPEN_REGION-regionserver/19945b63c2b4:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 40329a66ba4e990f8463ecffc5dfdaa9 2024-11-26T14:31:26,266 DEBUG [RS_OPEN_REGION-regionserver/19945b63c2b4:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1006): Region open journal for 40329a66ba4e990f8463ecffc5dfdaa9: Running coprocessor pre-open hook at 1732631486240Writing region info on filesystem at 1732631486241 (+1 ms)Initializing all the Stores at 1732631486242 (+1 ms)Instantiating store for column family {NAME => 'cf1', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732631486242Instantiating store for column family {NAME => 'cf2', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732631486242Cleaning up temporary data from old regions at 1732631486262 (+20 ms)Running coprocessor post-open hooks at 1732631486265 (+3 ms)Region opened successfully at 1732631486266 (+1 ms) 2024-11-26T14:31:26,267 INFO [RS_OPEN_REGION-regionserver/19945b63c2b4:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegionServer(2236): Post open deploy tasks for testReplayEditsAfterRegionMovedWithMultiCF,,1732631484097.40329a66ba4e990f8463ecffc5dfdaa9., pid=12, masterSystemTime=1732631486230 2024-11-26T14:31:26,269 DEBUG [RS_OPEN_REGION-regionserver/19945b63c2b4:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegionServer(2266): Finished post open deploy task for testReplayEditsAfterRegionMovedWithMultiCF,,1732631484097.40329a66ba4e990f8463ecffc5dfdaa9. 2024-11-26T14:31:26,269 INFO [RS_OPEN_REGION-regionserver/19945b63c2b4:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] handler.AssignRegionHandler(153): Opened testReplayEditsAfterRegionMovedWithMultiCF,,1732631484097.40329a66ba4e990f8463ecffc5dfdaa9. 2024-11-26T14:31:26,270 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=10 updating hbase:meta row=40329a66ba4e990f8463ecffc5dfdaa9, regionState=OPEN, openSeqNum=18, regionLocation=19945b63c2b4,35419,1732631469384 2024-11-26T14:31:26,272 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=12, ppid=10, state=RUNNABLE, hasLock=false; OpenRegionProcedure 40329a66ba4e990f8463ecffc5dfdaa9, server=19945b63c2b4,35419,1732631469384 because future has completed 2024-11-26T14:31:26,276 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=12, resume processing ppid=10 2024-11-26T14:31:26,277 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=12, ppid=10, state=SUCCESS, hasLock=false; OpenRegionProcedure 40329a66ba4e990f8463ecffc5dfdaa9, server=19945b63c2b4,35419,1732631469384 in 197 msec 2024-11-26T14:31:26,278 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=10, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=40329a66ba4e990f8463ecffc5dfdaa9, REOPEN/MOVE in 560 msec 2024-11-26T14:31:26,320 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-26T14:31:26,322 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:47664, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-26T14:31:26,324 ERROR [Time-limited test {}] regionserver.HRegionServer(2442): ***** ABORTING region server 19945b63c2b4,35419,1732631469384: testing ***** 2024-11-26T14:31:26,324 ERROR [Time-limited test {}] regionserver.HRegionServer(2447): RegionServer abort: loaded coprocessors are: [org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint] 2024-11-26T14:31:26,326 DEBUG [Time-limited test {}] util.JSONBean(135): Listing beans for java.lang:type=Memory 2024-11-26T14:31:26,327 DEBUG [Time-limited test {}] util.JSONBean(135): Listing beans for Hadoop:service=HBase,name=RegionServer,sub=IPC 2024-11-26T14:31:26,330 DEBUG [Time-limited test {}] util.JSONBean(135): Listing beans for Hadoop:service=HBase,name=RegionServer,sub=Replication 2024-11-26T14:31:26,331 DEBUG [Time-limited test {}] util.JSONBean(135): Listing beans for Hadoop:service=HBase,name=RegionServer,sub=Server 2024-11-26T14:31:26,339 INFO [Time-limited test {}] regionserver.HRegionServer(2451): Dump of metrics as JSON on abort: { "beans": [ { "name": "java.lang:type=Memory", "modelerType": "sun.management.MemoryImpl", "ObjectPendingFinalizationCount": 0, "HeapMemoryUsage": { "committed": 1048576000, "init": 1048576000, "max": 2306867200, "used": 315905856 }, "NonHeapMemoryUsage": { "committed": 170590208, "init": 7667712, "max": -1, "used": 167957088 }, "Verbose": false, "ObjectName": "java.lang:type=Memory" } ], "beans": [ { "name": "Hadoop:service=HBase,name=RegionServer,sub=IPC", "modelerType": "RegionServer,sub=IPC", "tag.Context": "regionserver", "tag.Hostname": "19945b63c2b4", "queueSize": 0, "numCallsInGeneralQueue": 0, "numCallsInReplicationQueue": 0, "numCallsInBulkLoadQueue": 0, "numCallsInPriorityQueue": 0, "numCallsInMetaPriorityQueue": 0, "numOpenConnections": 0, "numActiveHandler": 0, "numActiveGeneralHandler": 0, "numActivePriorityHandler": 0, "numActiveReplicationHandler": 0, "numGeneralCallsDropped": 0, "numLifoModeSwitches": 0, "numCallsInWriteQueue": 0, "numActiveBulkLoadHandler": 0, "numCallsInReadQueue": 0, "numCallsInScanQueue": 0, "numActiveWriteHandler": 0, "numActiveReadHandler": 0, "numActiveScanHandler": 0, "nettyDirectMemoryUsage": 67108864, "nettyTotalPendingOutboundBytes": 0, "nettyMaxPendingOutboundBytes": 0, "receivedBytes": 2071, "exceptions.RegionMovedException": 0, "authenticationSuccesses": 0, "authorizationFailures": 0, "exceptions.requestTooBig": 0, "UnwritableTime_num_ops": 0, "UnwritableTime_min": 0, "UnwritableTime_max": 0, "UnwritableTime_mean": 0, "UnwritableTime_25th_percentile": 0, "UnwritableTime_median": 0, "UnwritableTime_75th_percentile": 0, "UnwritableTime_90th_percentile": 0, "UnwritableTime_95th_percentile": 0, "UnwritableTime_98th_percentile": 0, "UnwritableTime_99th_percentile": 0, "UnwritableTime_99.9th_percentile": 0, "exceptions.OutOfOrderScannerNextException": 0, "exceptions.rpcThrottling": 0, "exceptions.otherExceptions": 0, "ProcessCallTime_num_ops": 8, "ProcessCallTime_min": 0, "ProcessCallTime_max": 27, "ProcessCallTime_mean": 7, "ProcessCallTime_25th_percentile": 6, "ProcessCallTime_median": 13, "ProcessCallTime_75th_percentile": 20, "ProcessCallTime_90th_percentile": 24, "ProcessCallTime_95th_percentile": 25, "ProcessCallTime_98th_percentile": 26, "ProcessCallTime_99th_percentile": 26, "ProcessCallTime_99.9th_percentile": 26, "ProcessCallTime_TimeRangeCount_0-1": 8, "exceptions.callQueueTooBig": 0, "QueueCallTime_num_ops": 8, "QueueCallTime_min": 0, "QueueCallTime_max": 1, "QueueCallTime_mean": 0, "QueueCallTime_25th_percentile": 0, "QueueCallTime_median": 0, "QueueCallTime_75th_percentile": 0, "QueueCallTime_90th_percentile": 0, "QueueCallTime_95th_percentile": 0, "QueueCallTime_98th_percentile": 0, "QueueCallTime_99th_percentile": 0, "QueueCallTime_99.9th_percentile": 0, "QueueCallTime_TimeRangeCount_0-1": 8, "authenticationFailures": 0, "exceptions.multiResponseTooLarge": 0, "exceptions.callDropped": 0, "TotalCallTime_num_ops": 8, "TotalCallTime_min": 0, "TotalCallTime_max": 27, "TotalCallTime_mean": 7, "TotalCallTime_25th_percentile": 6, "TotalCallTime_median": 13, "TotalCallTime_75th_percentile": 20, "TotalCallTime_90th_percentile": 24, "TotalCallTime_95th_percentile": 25, "TotalCallTime_98th_percentile": 26, "TotalCallTime_99th_percentile": 26, "TotalCallTime_99.9th_percentile": 26, "TotalCallTime_TimeRangeCount_0-1": 8, "exceptions.RegionTooBusyException": 0, "exceptions.FailedSanityCheckException": 0, "ResponseSize_num_ops": 8, "ResponseSize_min": 0, "ResponseSize_max": 174, "ResponseSize_mean": 50, "ResponseSize_25th_percentile": 43, "ResponseSize_median": 87, "ResponseSize_75th_percentile": 130, "ResponseSize_90th_percentile": 156, "ResponseSize_95th_percentile": 165, "ResponseSize_98th_percentile": 170, "ResponseSize_99th_percentile": 172, "ResponseSize_99.9th_percentile": 173, "ResponseSize_SizeRangeCount_0-10": 8, "exceptions.UnknownScannerException": 0, "exceptions": 0, "maxOutboundBytesExceeded": 0, "authenticationFallbacks": 0, "exceptions.quotaExceeded": 0, "exceptions.callTimedOut": 0, "exceptions.NotServingRegionException": 0, "authorizationSuccesses": 0, "exceptions.ScannerResetException": 0, "RequestSize_num_ops": 8, "RequestSize_min": 89, "RequestSize_max": 121, "RequestSize_mean": 103, "RequestSize_25th_percentile": 97, "RequestSize_median": 105, "RequestSize_75th_percentile": 113, "RequestSize_90th_percentile": 117, "RequestSize_95th_percentile": 119, "RequestSize_98th_percentile": 120, "RequestSize_99th_percentile": 120, "RequestSize_99.9th_percentile": 120, "RequestSize_SizeRangeCount_0-10": 8, "sentBytes": 348 } ], "beans": [ { "name": "Hadoop:service=HBase,name=RegionServer,sub=Replication", "modelerType": "RegionServer,sub=Replication", "tag.Context": "regionserver", "tag.Hostname": "19945b63c2b4", "source.shippedHFiles": 0, "Source.ageOfLastShippedOp_num_ops": 0, "Source.ageOfLastShippedOp_min": 0, "Source.ageOfLastShippedOp_max": 0, "Source.ageOfLastShippedOp_mean": 0, "Source.ageOfLastShippedOp_25th_percentile": 0, "Source.ageOfLastShippedOp_median": 0, "Source.ageOfLastShippedOp_75th_percentile": 0, "Source.ageOfLastShippedOp_90th_percentile": 0, "Source.ageOfLastShippedOp_95th_percentile": 0, "Source.ageOfLastShippedOp_98th_percentile": 0, "Source.ageOfLastShippedOp_99th_percentile": 0, "Source.ageOfLastShippedOp_99.9th_percentile": 0, "source.uncleanlyClosedLogs": 0, "source.closedLogsWithUnknownFileLength": 0, "source.walReaderEditsBufferUsage": 0, "source.repeatedLogFileBytes": 0, "source.sizeOfHFileRefsQueue": 0, "source.logReadInBytes": 0, "source.completedRecoverQueues": 0, "source.sizeOfLogQueue": 0, "source.restartedLogReading": 0, "source.failedRecoverQueues": 0, "source.ignoredUncleanlyClosedLogContentsInBytes": 0, "Sink.ageOfLastAppliedOp_num_ops": 0, "Sink.ageOfLastAppliedOp_min": 0, "Sink.ageOfLastAppliedOp_max": 0, "Sink.ageOfLastAppliedOp_mean": 0, "Sink.ageOfLastAppliedOp_25th_percentile": 0, "Sink.ageOfLastAppliedOp_median": 0, "Sink.ageOfLastAppliedOp_75th_percentile": 0, "Sink.ageOfLastAppliedOp_90th_percentile": 0, "Sink.ageOfLastAppliedOp_95th_percentile": 0, "Sink.ageOfLastAppliedOp_98th_percentile": 0, "Sink.ageOfLastAppliedOp_99th_percentile": 0, "Sink.ageOfLastAppliedOp_99.9th_percentile": 0, "source.logEditsRead": 0, "source.numInitializing": 0, "source.shippedOps": 0, "sink.appliedHFiles": 0, "source.logEditsFiltered": 0, "source.shippedBytes": 0, "sink.appliedOps": 0, "source.completedLogs": 0, "source.failedBatches": 0, "sink.failedBatches": 0, "source.shippedBatches": 0, "sink.appliedBatches": 0 } ], "beans": [ { "name": "Hadoop:service=HBase,name=RegionServer,sub=Server", "modelerType": "RegionServer,sub=Server", "tag.zookeeperQuorum": "127.0.0.1:49403", "tag.serverName": "19945b63c2b4,37467,1732631469538", "tag.clusterId": "85e6cd0d-a40d-403d-9d6e-04753ee5ca5e", "tag.Context": "regionserver", "tag.Hostname": "19945b63c2b4", "regionCount": 1, "storeCount": 4, "hlogFileCount": 2, "hlogFileSize": 0, "storeFileCount": 0, "maxStoreFileCount": 0, "memStoreSize": 74, "memStoreHeapSize": 1248, "memStoreOffHeapSize": 0, "storeFileSize": 0, "storeFileSizeGrowthRate": 0.0, "maxStoreFileAge": 0, "minStoreFileAge": 0, "avgStoreFileAge": 0, "numReferenceFiles": 0, "regionServerStartTime": 1732631469538, "averageRegionSize": 74, "storeFileIndexSize": 0, "staticIndexSize": 0, "staticBloomSize": 0, "bloomFilterRequestsCount": 0, "bloomFilterNegativeResultsCount": 0, "bloomFilterEligibleRequestsCount": 0, "mutationsWithoutWALCount": 0, "mutationsWithoutWALSize": 0, "percentFilesLocal": 0.0, "percentFilesLocalSecondaryRegions": 0.0, "totalBytesRead": 10034, "localBytesRead": 10034, "shortCircuitBytesRead": 0, "zeroCopyBytesRead": 0, "splitQueueLength": 0, "compactionQueueLength": 0, "smallCompactionQueueLength": 0, "largeCompactionQueueLength": 0, "flushQueueLength": 0, "blockCacheFreeSize": 922070024, "blockCacheCount": 0, "blockCacheDataBlockCount": 0, "blockCacheSize": 676856, "blockCacheCountHitPercent": 0.0, "blockCacheExpressHitPercent": 0.0, "l1CacheSize": 676856, "l1CacheFreeSize": 922070024, "l1CacheCount": 0, "l1CacheEvictionCount": 0, "l1CacheHitCount": 0, "l1CacheMissCount": 0, "l1CacheHitRatio": 0.0, "l1CacheMissRatio": 0.0, "l2CacheSize": 0, "l2CacheFreeSize": 0, "l2CacheCount": 0, "l2CacheEvictionCount": 0, "l2CacheHitCount": 0, "l2CacheMissCount": 0, "l2CacheHitRatio": 0.0, "l2CacheMissRatio": 0.0, "mobFileCacheCount": 0, "mobFileCacheHitPercent": 0.0, "readRequestRatePerSecond": 1.4, "writeRequestRatePerSecond": 0.4, "ByteBuffAllocatorHeapAllocationBytes": 4782, "ByteBuffAllocatorPoolAllocationBytes": 199680, "ByteBuffAllocatorHeapAllocationRatio": 0.0, "ByteBuffAllocatorTotalBufferCount": 186, "ByteBuffAllocatorUsedBufferCount": 1, "activeScanners": 0, "totalRequestCount": 7, "totalRowActionRequestCount": 9, "readRequestCount": 7, "cpRequestCount": 0, "filteredReadRequestCount": 0, "writeRequestCount": 2, "rpcGetRequestCount": 1, "rpcFullScanRequestCount": 0, "rpcScanRequestCount": 4, "rpcMultiRequestCount": 0, "rpcMutateRequestCount": 2, "checkMutateFailedCount": 0, "checkMutatePassedCount": 0, "blockCacheHitCount": 0, "blockCacheHitCountPrimary": 0, "blockCacheHitCachingCount": 0, "blockCacheMissCount": 0, "blockCacheMissCountPrimary": 0, "blockCacheMissCachingCount": 0, "blockCacheEvictionCount": 0, "blockCacheEvictionCountPrimary": 0, "blockCacheFailedInsertionCount": 0, "blockCacheDataMissCount": 0, "blockCacheLeafIndexMissCount": 0, "blockCacheBloomChunkMissCount": 0, "blockCacheMetaMissCount": 0, "blockCacheRootIndexMissCount": 0, "blockCacheIntermediateIndexMissCount": 0, "blockCacheFileInfoMissCount": 0, "blockCacheGeneralBloomMetaMissCount": 0, "blockCacheDeleteFamilyBloomMissCount": 0, "blockCacheTrailerMissCount": 0, "blockCacheDataHitCount": 0, "blockCacheLeafIndexHitCount": 0, "blockCacheBloomChunkHitCount": 0, "blockCacheMetaHitCount": 0, "blockCacheRootIndexHitCount": 0, "blockCacheIntermediateIndexHitCount": 0, "blockCacheFileInfoHitCount": 0, "blockCacheGeneralBloomMetaHitCount": 0, "blockCacheDeleteFamilyBloomHitCount": 0, "blockCacheTrailerHitCount": 0, "updatesBlockedTime": 0, "flushedCellsCount": 0, "compactedCellsCount": 0, "majorCompactedCellsCount": 0, "flushedCellsSize": 0, "compactedCellsSize": 0, "majorCompactedCellsSize": 0, "cellsCountCompactedFromMob": 0, "cellsCountCompactedToMob": 0, "cellsSizeCompactedFromMob": 0, "cellsSizeCompactedToMob": 0, "mobFlushCount": 0, "mobFlushedCellsCount": 0, "mobFlushedCellsSize": 0, "mobScanCellsCount": 0, "mobScanCellsSize": 0, "mobFileCacheAccessCount": 0, "mobFileCacheMissCount": 0, "mobFileCacheEvictedCount": 0, "hedgedReads": 0, "hedgedReadWins": 0, "hedgedReadOpsInCurThread": 0, "blockedRequestCount": 0, "CheckAndMutate_num_ops": 0, "CheckAndMutate_min": 0, "CheckAndMutate_max": 0, "CheckAndMutate_mean": 0, "CheckAndMutate_25th_percentile": 0, "CheckAndMutate_median": 0, "CheckAndMutate_75th_percentile": 0, "CheckAndMutate_90th_percentile": 0, "CheckAndMutate_95th_percentile": 0, "CheckAndMutate_98th_percentile": 0, "CheckAndMutate_99th_percentile": 0, "CheckAndMutate_99.9th_percentile": 0, "MajorCompactionTime_num_ops": 0, "MajorCompactionTime_min": 0, "MajorCompactionTime_max": 0, "MajorCompactionTime_mean": 0, "MajorCompactionTime_25th_percentile": 0, "MajorCompactionTime_median": 0, "MajorCompactionTime_75th_percentile": 0, "MajorCompactionTime_90th_percentile": 0, "MajorCompactionTime_95th_percentile": 0, "MajorCompactionTime_98th_percentile": 0, "MajorCompactionTime_99th_percentile": 0, "MajorCompactionTime_99.9th_percentile": 0, "ScanTime_num_ops": 4, "ScanTime_min": 0, "ScanTime_max": 2, "ScanTime_mean": 0, "ScanTime_25th_percentile": 0, "ScanTime_median": 1, "ScanTime_75th_percentile": 1, "ScanTime_90th_percentile": 1, "ScanTime_95th_percentile": 1, "ScanTime_98th_percentile": 1, "ScanTime_99th_percentile": 1, "ScanTime_99.9th_percentile": 1, "ScanTime_TimeRangeCount_0-1": 4, "CheckAndMutateBlockBytesScanned_num_ops": 0, "CheckAndMutateBlockBytesScanned_min": 0, "CheckAndMutateBlockBytesScanned_max": 0, "CheckAndMutateBlockBytesScanned_mean": 0, "CheckAndMutateBlockBytesScanned_25th_percentile": 0, "CheckAndMutateBlockBytesScanned_median": 0, "CheckAndMutateBlockBytesScanned_75th_percentile": 0, "CheckAndMutateBlockBytesScanned_90th_percentile": 0, "CheckAndMutateBlockBytesScanned_95th_percentile": 0, "CheckAndMutateBlockBytesScanned_98th_percentile": 0, "CheckAndMutateBlockBytesScanned_99th_percentile": 0, "CheckAndMutateBlockBytesScanned_99.9th_percentile": 0, "Put_num_ops": 2, "Put_min": 2, "Put_max": 7, "Put_mean": 4, "Put_25th_percentile": 3, "Put_median": 4, "Put_75th_percentile": 5, "Put_90th_percentile": 6, "Put_95th_percentile": 6, "Put_98th_percentile": 6, "Put_99th_percentile": 6, "Put_99.9th_percentile": 6, "Put_TimeRangeCount_0-1": 2, "splitRequestCount": 0, "AppendBlockBytesScanned_num_ops": 0, "AppendBlockBytesScanned_min": 0, "AppendBlockBytesScanned_max": 0, "AppendBlockBytesScanned_mean": 0, "AppendBlockBytesScanned_25th_percentile": 0, "AppendBlockBytesScanned_median": 0, "AppendBlockBytesScanned_75th_percentile": 0, "AppendBlockBytesScanned_90th_percentile": 0, "AppendBlockBytesScanned_95th_percentile": 0, "AppendBlockBytesScanned_98th_percentile": 0, "AppendBlockBytesScanned_99th_percentile": 0, "AppendBlockBytesScanned_99.9th_percentile": 0, "PutBatch_num_ops": 0, "PutBatch_min": 0, "PutBatch_max": 0, "PutBatch_mean": 0, "PutBatch_25th_percentile": 0, "PutBatch_median": 0, "PutBatch_75th_percentile": 0, "PutBatch_90th_percentile": 0, "PutBatch_95th_percentile": 0, "PutBatch_98th_percentile": 0, "PutBatch_99th_percentile": 0, "PutBatch_99.9th_percentile": 0, "IncrementBlockBytesScanned_num_ops": 0, "IncrementBlockBytesScanned_min": 0, "IncrementBlockBytesScanned_max": 0, "IncrementBlockBytesScanned_mean": 0, "IncrementBlockBytesScanned_25th_percentile": 0, "IncrementBlockBytesScanned_median": 0, "IncrementBlockBytesScanned_75th_percentile": 0, "IncrementBlockBytesScanned_90th_percentile": 0, "IncrementBlockBytesScanned_95th_percentile": 0, "IncrementBlockBytesScanned_98th_percentile": 0, "IncrementBlockBytesScanned_99th_percentile": 0, "IncrementBlockBytesScanned_99.9th_percentile": 0, "SplitTime_num_ops": 0, "SplitTime_min": 0, "SplitTime_max": 0, "SplitTime_mean": 0, "SplitTime_25th_percentile": 0, "SplitTime_median": 0, "SplitTime_75th_percentile": 0, "SplitTime_90th_percentile": 0, "SplitTime_95th_percentile": 0, "SplitTime_98th_percentile": 0, "SplitTime_99th_percentile": 0, "SplitTime_99.9th_percentile": 0, "GetBlockBytesScanned_num_ops": 0, "GetBlockBytesScanned_min": 0, "GetBlockBytesScanned_max": 0, "GetBlockBytesScanned_mean": 0, "GetBlockBytesScanned_25th_percentile": 0, "GetBlockBytesScanned_median": 0, "GetBlockBytesScanned_75th_percentile": 0, "GetBlockBytesScanned_90th_percentile": 0, "GetBlockBytesScanned_95th_percentile": 0, "GetBlockBytesScanned_98th_percentile": 0, "GetBlockBytesScanned_99th_percentile": 0, "GetBlockBytesScanned_99.9th_percentile": 0, "majorCompactedInputBytes": 0, "slowAppendCount": 0, "flushedOutputBytes": 0, "Replay_num_ops": 0, "Replay_min": 0, "Replay_max": 0, "Replay_mean": 0, "Replay_25th_percentile": 0, "Replay_median": 0, "Replay_75th_percentile": 0, "Replay_90th_percentile": 0, "Replay_95th_percentile": 0, "Replay_98th_percentile": 0, "Replay_99th_percentile": 0, "Replay_99.9th_percentile": 0, "MajorCompactionInputSize_num_ops": 0, "MajorCompactionInputSize_min": 0, "MajorCompactionInputSize_max": 0, "MajorCompactionInputSize_mean": 0, "MajorCompactionInputSize_25th_percentile": 0, "MajorCompactionInputSize_median": 0, "MajorCompactionInputSize_75th_percentile": 0, "MajorCompactionInputSize_90th_percentile": 0, "MajorCompactionInputSize_95th_percentile": 0, "MajorCompactionInputSize_98th_percentile": 0, "MajorCompactionInputSize_99th_percentile": 0, "MajorCompactionInputSize_99.9th_percentile": 0, "pauseInfoThresholdExceeded": 0, "CheckAndDelete_num_ops": 0, "CheckAndDelete_min": 0, "CheckAndDelete_max": 0, "CheckAndDelete_mean": 0, "CheckAndDelete_25th_percentile": 0, "CheckAndDelete_median": 0, "CheckAndDelete_75th_percentile": 0, "CheckAndDelete_90th_percentile": 0, "CheckAndDelete_95th_percentile": 0, "CheckAndDelete_98th_percentile": 0, "CheckAndDelete_99th_percentile": 0, "CheckAndDelete_99.9th_percentile": 0, "CompactionInputSize_num_ops": 0, "CompactionInputSize_min": 0, "CompactionInputSize_max": 0, "CompactionInputSize_mean": 0, "CompactionInputSize_25th_percentile": 0, "CompactionInputSize_median": 0, "CompactionInputSize_75th_percentile": 0, "CompactionInputSize_90th_percentile": 0, "CompactionInputSize_95th_percentile": 0, "CompactionInputSize_98th_percentile": 0, "CompactionInputSize_99th_percentile": 0, "CompactionInputSize_99.9th_percentile": 0, "flushedMemstoreBytes": 0, "majorCompactedOutputBytes": 0, "slowPutCount": 0, "compactedInputBytes": 0, "FlushOutputSize_num_ops": 0, "FlushOutputSize_min": 0, "FlushOutputSize_max": 0, "FlushOutputSize_mean": 0, "FlushOutputSize_25th_percentile": 0, "FlushOutputSize_median": 0, "FlushOutputSize_75th_percentile": 0, "FlushOutputSize_90th_percentile": 0, "FlushOutputSize_95th_percentile": 0, "FlushOutputSize_98th_percentile": 0, "FlushOutputSize_99th_percentile": 0, "FlushOutputSize_99.9th_percentile": 0, "PauseTimeWithGc_num_ops": 0, "PauseTimeWithGc_min": 0, "PauseTimeWithGc_max": 0, "PauseTimeWithGc_mean": 0, "PauseTimeWithGc_25th_percentile": 0, "PauseTimeWithGc_median": 0, "PauseTimeWithGc_75th_percentile": 0, "PauseTimeWithGc_90th_percentile": 0, "PauseTimeWithGc_95th_percentile": 0, "PauseTimeWithGc_98th_percentile": 0, "PauseTimeWithGc_99th_percentile": 0, "PauseTimeWithGc_99.9th_percentile": 0, "compactedOutputBytes": 0, "pauseWarnThresholdExceeded": 0, "ScanBlockBytesScanned_num_ops": 0, "ScanBlockBytesScanned_min": 0, "ScanBlockBytesScanned_max": 0, "ScanBlockBytesScanned_mean": 0, "ScanBlockBytesScanned_25th_percentile": 0, "ScanBlockBytesScanned_median": 0, "ScanBlockBytesScanned_75th_percentile": 0, "ScanBlockBytesScanned_90th_percentile": 0, "ScanBlockBytesScanned_95th_percentile": 0, "ScanBlockBytesScanned_98th_percentile": 0, "ScanBlockBytesScanned_99th_percentile": 0, "ScanBlockBytesScanned_99.9th_percentile": 0, "Increment_num_ops": 0, "Increment_min": 0, "Increment_max": 0, "Increment_mean": 0, "Increment_25th_percentile": 0, "Increment_median": 0, "Increment_75th_percentile": 0, "Increment_90th_percentile": 0, "Increment_95th_percentile": 0, "Increment_98th_percentile": 0, "Increment_99th_percentile": 0, "Increment_99.9th_percentile": 0, "Delete_num_ops": 0, "Delete_min": 0, "Delete_max": 0, "Delete_mean": 0, "Delete_25th_percentile": 0, "Delete_median": 0, "Delete_75th_percentile": 0, "Delete_90th_percentile": 0, "Delete_95th_percentile": 0, "Delete_98th_percentile": 0, "Delete_99th_percentile": 0, "Delete_99.9th_percentile": 0, "DeleteBatch_num_ops": 0, "DeleteBatch_min": 0, "DeleteBatch_max": 0, "DeleteBatch_mean": 0, "DeleteBatch_25th_percentile": 0, "DeleteBatch_median": 0, "DeleteBatch_75th_percentile": 0, "DeleteBatch_90th_percentile": 0, "DeleteBatch_95th_percentile": 0, "DeleteBatch_98th_percentile": 0, "DeleteBatch_99th_percentile": 0, "DeleteBatch_99.9th_percentile": 0, "blockBytesScannedCount": 0, "FlushMemstoreSize_num_ops": 0, "FlushMemstoreSize_min": 0, "FlushMemstoreSize_max": 0, "FlushMemstoreSize_mean": 0, "FlushMemstoreSize_25th_percentile": 0, "FlushMemstoreSize_median": 0, "FlushMemstoreSize_75th_percentile": 0, "FlushMemstoreSize_90th_percentile": 0, "FlushMemstoreSize_95th_percentile": 0, "FlushMemstoreSize_98th_percentile": 0, "FlushMemstoreSize_99th_percentile": 0, "FlushMemstoreSize_99.9th_percentile": 0, "CompactionInputFileCount_num_ops": 0, "CompactionInputFileCount_min": 0, "CompactionInputFileCount_max": 0, "CompactionInputFileCount_mean": 0, "CompactionInputFileCount_25th_percentile": 0, "CompactionInputFileCount_median": 0, "CompactionInputFileCount_75th_percentile": 0, "CompactionInputFileCount_90th_percentile": 0, "CompactionInputFileCount_95th_percentile": 0, "CompactionInputFileCount_98th_percentile": 0, "CompactionInputFileCount_99th_percentile": 0, "CompactionInputFileCount_99.9th_percentile": 0, "CompactionTime_num_ops": 0, "CompactionTime_min": 0, "CompactionTime_max": 0, "CompactionTime_mean": 0, "CompactionTime_25th_percentile": 0, "CompactionTime_median": 0, "CompactionTime_75th_percentile": 0, "CompactionTime_90th_percentile": 0, "CompactionTime_95th_percentile": 0, "CompactionTime_98th_percentile": 0, "CompactionTime_99th_percentile": 0, "CompactionTime_99.9th_percentile": 0, "Get_num_ops": 1, "Get_min": 5, "Get_max": 5, "Get_mean": 5, "Get_25th_percentile": 5, "Get_median": 5, "Get_75th_percentile": 5, "Get_90th_percentile": 5, "Get_95th_percentile": 5, "Get_98th_percentile": 5, "Get_99th_percentile": 5, "Get_99.9th_percentile": 5, "Get_TimeRangeCount_0-1": 1, "MajorCompactionInputFileCount_num_ops": 0, "MajorCompactionInputFileCount_min": 0, "MajorCompactionInputFileCount_max": 0, "MajorCompactionInputFileCount_mean": 0, "MajorCompactionInputFileCount_25th_percentile": 0, "MajorCompactionInputFileCount_median": 0, "MajorCompactionInputFileCount_75th_percentile": 0, "MajorCompactionInputFileCount_90th_percentile": 0, "MajorCompactionInputFileCount_95th_percentile": 0, "MajorCompactionInputFileCount_98th_percentile": 0, "MajorCompactionInputFileCount_99th_percentile": 0, "MajorCompactionInputFileCount_99.9th_percentile": 0, "scannerLeaseExpiredCount": 0, "CheckAndPut_num_ops": 0, "CheckAndPut_min": 0, "CheckAndPut_max": 0, "CheckAndPut_mean": 0, "CheckAndPut_25th_percentile": 0, "CheckAndPut_median": 0, "CheckAndPut_75th_percentile": 0, "CheckAndPut_90th_percentile": 0, "CheckAndPut_95th_percentile": 0, "CheckAndPut_98th_percentile": 0, "CheckAndPut_99th_percentile": 0, "CheckAndPut_99.9th_percentile": 0, "MajorCompactionOutputSize_num_ops": 0, "MajorCompactionOutputSize_min": 0, "MajorCompactionOutputSize_max": 0, "MajorCompactionOutputSize_mean": 0, "MajorCompactionOutputSize_25th_percentile": 0, "MajorCompactionOutputSize_median": 0, "MajorCompactionOutputSize_75th_percentile": 0, "MajorCompactionOutputSize_90th_percentile": 0, "MajorCompactionOutputSize_95th_percentile": 0, "MajorCompactionOutputSize_98th_percentile": 0, "MajorCompactionOutputSize_99th_percentile": 0, "MajorCompactionOutputSize_99.9th_percentile": 0, "CompactionOutputFileCount_num_ops": 0, "CompactionOutputFileCount_min": 0, "CompactionOutputFileCount_max": 0, "CompactionOutputFileCount_mean": 0, "CompactionOutputFileCount_25th_percentile": 0, "CompactionOutputFileCount_median": 0, "CompactionOutputFileCount_75th_percentile": 0, "CompactionOutputFileCount_90th_percentile": 0, "CompactionOutputFileCount_95th_percentile": 0, "CompactionOutputFileCount_98th_percentile": 0, "CompactionOutputFileCount_99th_percentile": 0, "CompactionOutputFileCount_99.9th_percentile": 0, "slowDeleteCount": 0, "FlushTime_num_ops": 0, "FlushTime_min": 0, "FlushTime_max": 0, "FlushTime_mean": 0, "FlushTime_25th_percentile": 0, "FlushTime_median": 0, "FlushTime_75th_percentile": 0, "FlushTime_90th_percentile": 0, "FlushTime_95th_percentile": 0, "FlushTime_98th_percentile": 0, "FlushTime_99th_percentile": 0, "FlushTime_99.9th_percentile": 0, "splitSuccessCount": 0, "MajorCompactionOutputFileCount_num_ops": 0, "MajorCompactionOutputFileCount_min": 0, "MajorCompactionOutputFileCount_max": 0, "MajorCompactionOutputFileCount_mean": 0, "MajorCompactionOutputFileCount_25th_percentile": 0, "MajorCompactionOutputFileCount_median": 0, "MajorCompactionOutputFileCount_75th_percentile": 0, "MajorCompactionOutputFileCount_90th_percentile": 0, "MajorCompactionOutputFileCount_95th_percentile": 0, "MajorCompactionOutputFileCount_98th_percentile": 0, "MajorCompactionOutputFileCount_99th_percentile": 0, "MajorCompactionOutputFileCount_99.9th_percentile": 0, "slowGetCount": 0, "ScanSize_num_ops": 4, "ScanSize_min": 0, "ScanSize_max": 144, "ScanSize_mean": 72, "ScanSize_25th_percentile": 36, "ScanSize_median": 72, "ScanSize_75th_percentile": 108, "ScanSize_90th_percentile": 129, "ScanSize_95th_percentile": 136, "ScanSize_98th_percentile": 141, "ScanSize_99th_percentile": 142, "ScanSize_99.9th_percentile": 143, "ScanSize_SizeRangeCount_0-10": 4, "CompactionOutputSize_num_ops": 0, "CompactionOutputSize_min": 0, "CompactionOutputSize_max": 0, "CompactionOutputSize_mean": 0, "CompactionOutputSize_25th_percentile": 0, "CompactionOutputSize_median": 0, "CompactionOutputSize_75th_percentile": 0, "CompactionOutputSize_90th_percentile": 0, "CompactionOutputSize_95th_percentile": 0, "CompactionOutputSize_98th_percentile": 0, "CompactionOutputSize_99th_percentile": 0, "CompactionOutputSize_99.9th_percentile": 0, "PauseTimeWithoutGc_num_ops": 0, "PauseTimeWithoutGc_min": 0, "PauseTimeWithoutGc_max": 0, "PauseTimeWithoutGc_mean": 0, "PauseTimeWithoutGc_25th_percentile": 0, "PauseTimeWithoutGc_median": 0, "PauseTimeWithoutGc_75th_percentile": 0, "PauseTimeWithoutGc_90th_percentile": 0, "PauseTimeWithoutGc_95th_percentile": 0, "PauseTimeWithoutGc_98th_percentile": 0, "PauseTimeWithoutGc_99th_percentile": 0, "PauseTimeWithoutGc_99.9th_percentile": 0, "slowIncrementCount": 0, "Append_num_ops": 0, "Append_min": 0, "Append_max": 0, "Append_mean": 0, "Append_25th_percentile": 0, "Append_median": 0, "Append_75th_percentile": 0, "Append_90th_percentile": 0, "Append_95th_percentile": 0, "Append_98th_percentile": 0, "Append_99th_percentile": 0, "Append_99.9th_percentile": 0, "Bulkload_count": 0, "Bulkload_mean_rate": 0.0, "Bulkload_1min_rate": 0.0, "Bulkload_5min_rate": 0.0, "Bulkload_15min_rate": 0.0, "Bulkload_num_ops": 0, "Bulkload_min": 0, "Bulkload_max": 0, "Bulkload_mean": 0, "Bulkload_25th_percentile": 0, "Bulkload_median": 0, "Bulkload_75th_percentile": 0, "Bulkload_90th_percentile": 0, "Bulkload_95th_percentile": 0, "Bulkload_98th_percentile": 0, "Bulkload_99th_percentile": 0, "Bulkload_99.9th_percentile": 0 } ] } 2024-11-26T14:31:26,343 WARN [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43577 {}] master.MasterRpcServices(700): 19945b63c2b4,35419,1732631469384 reported a fatal error: ***** ABORTING region server 19945b63c2b4,35419,1732631469384: testing ***** 2024-11-26T14:31:26,345 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '19945b63c2b4,35419,1732631469384' ***** 2024-11-26T14:31:26,345 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: testing 2024-11-26T14:31:26,346 INFO [RS:0;19945b63c2b4:35419 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-26T14:31:26,346 INFO [RS:0;19945b63c2b4:35419 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager abruptly. 2024-11-26T14:31:26,346 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-26T14:31:26,346 INFO [RS:0;19945b63c2b4:35419 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager abruptly. 2024-11-26T14:31:26,346 INFO [RS:0;19945b63c2b4:35419 {}] regionserver.HRegionServer(3091): Received CLOSE for 40329a66ba4e990f8463ecffc5dfdaa9 2024-11-26T14:31:26,347 INFO [RS:0;19945b63c2b4:35419 {}] regionserver.HRegionServer(956): aborting server 19945b63c2b4,35419,1732631469384 2024-11-26T14:31:26,347 INFO [RS:0;19945b63c2b4:35419 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-26T14:31:26,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40179 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Get size: 140 connection: 172.17.0.3:34830 deadline: 1732631546347, exception=org.apache.hadoop.hbase.exceptions.RegionMovedException: Region moved to: hostname=19945b63c2b4 port=35419 startCode=1732631469384. As of locationSeqNum=12. 2024-11-26T14:31:26,347 INFO [RS:0;19945b63c2b4:35419 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;19945b63c2b4:35419. 2024-11-26T14:31:26,347 DEBUG [RS_CLOSE_REGION-regionserver/19945b63c2b4:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 40329a66ba4e990f8463ecffc5dfdaa9, disabling compactions & flushes 2024-11-26T14:31:26,347 INFO [RS_CLOSE_REGION-regionserver/19945b63c2b4:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testReplayEditsAfterRegionMovedWithMultiCF,,1732631484097.40329a66ba4e990f8463ecffc5dfdaa9. 2024-11-26T14:31:26,347 DEBUG [RS:0;19945b63c2b4:35419 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-26T14:31:26,347 DEBUG [RS_CLOSE_REGION-regionserver/19945b63c2b4:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1732631484097.40329a66ba4e990f8463ecffc5dfdaa9. 2024-11-26T14:31:26,347 DEBUG [RS_CLOSE_REGION-regionserver/19945b63c2b4:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1732631484097.40329a66ba4e990f8463ecffc5dfdaa9. after waiting 0 ms 2024-11-26T14:31:26,347 DEBUG [RS_CLOSE_REGION-regionserver/19945b63c2b4:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsAfterRegionMovedWithMultiCF,,1732631484097.40329a66ba4e990f8463ecffc5dfdaa9. 2024-11-26T14:31:26,348 DEBUG [RS:0;19945b63c2b4:35419 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T14:31:26,348 DEBUG [RPCClient-NioEventLoopGroup-6-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1732631484097.40329a66ba4e990f8463ecffc5dfdaa9., hostname=19945b63c2b4,40179,1732631469496, seqNum=5 , the old value is region=testReplayEditsAfterRegionMovedWithMultiCF,,1732631484097.40329a66ba4e990f8463ecffc5dfdaa9., hostname=19945b63c2b4,40179,1732631469496, seqNum=5, error=org.apache.hadoop.hbase.exceptions.RegionMovedException: Region moved to: hostname=19945b63c2b4 port=35419 startCode=1732631469384. As of locationSeqNum=12. 2024-11-26T14:31:26,348 DEBUG [RPCClient-NioEventLoopGroup-6-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1732631484097.40329a66ba4e990f8463ecffc5dfdaa9., hostname=19945b63c2b4,40179,1732631469496, seqNum=5 is org.apache.hadoop.hbase.exceptions.RegionMovedException: Region moved to: hostname=19945b63c2b4 port=35419 startCode=1732631469384. As of locationSeqNum=12. 2024-11-26T14:31:26,348 DEBUG [RPCClient-NioEventLoopGroup-6-7 {}] client.AsyncRegionLocatorHelper(84): Try updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1732631484097.40329a66ba4e990f8463ecffc5dfdaa9., hostname=19945b63c2b4,40179,1732631469496, seqNum=5 with the new location region=testReplayEditsAfterRegionMovedWithMultiCF,,1732631484097.40329a66ba4e990f8463ecffc5dfdaa9., hostname=19945b63c2b4,35419,1732631469384, seqNum=12 constructed by org.apache.hadoop.hbase.exceptions.RegionMovedException: Region moved to: hostname=19945b63c2b4 port=35419 startCode=1732631469384. As of locationSeqNum=12. 2024-11-26T14:31:26,348 INFO [RS:0;19945b63c2b4:35419 {}] regionserver.HRegionServer(1321): Waiting on 1 regions to close 2024-11-26T14:31:26,348 DEBUG [RS:0;19945b63c2b4:35419 {}] regionserver.HRegionServer(1325): Online Regions={40329a66ba4e990f8463ecffc5dfdaa9=testReplayEditsAfterRegionMovedWithMultiCF,,1732631484097.40329a66ba4e990f8463ecffc5dfdaa9.} 2024-11-26T14:31:26,349 DEBUG [RS:0;19945b63c2b4:35419 {}] regionserver.HRegionServer(1351): Waiting on 40329a66ba4e990f8463ecffc5dfdaa9 2024-11-26T14:31:26,350 INFO [RS_CLOSE_REGION-regionserver/19945b63c2b4:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testReplayEditsAfterRegionMovedWithMultiCF,,1732631484097.40329a66ba4e990f8463ecffc5dfdaa9. 2024-11-26T14:31:26,351 DEBUG [RS_CLOSE_REGION-regionserver/19945b63c2b4:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 40329a66ba4e990f8463ecffc5dfdaa9: Waiting for close lock at 1732631486347Running coprocessor pre-close hooks at 1732631486347Disabling compacts and flushes for region at 1732631486347Disabling writes for close at 1732631486347Writing region close event to WAL at 1732631486350 (+3 ms)Running coprocessor post-close hooks at 1732631486350Closed at 1732631486350 2024-11-26T14:31:26,351 DEBUG [RS_CLOSE_REGION-regionserver/19945b63c2b4:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testReplayEditsAfterRegionMovedWithMultiCF,,1732631484097.40329a66ba4e990f8463ecffc5dfdaa9. 2024-11-26T14:31:26,392 INFO [regionserver/19945b63c2b4:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-26T14:31:26,457 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.regionserver.RegionServerAbortedException: Server 19945b63c2b4,35419,1732631469384 aborting at org.apache.hadoop.hbase.ipc.ServerRpcConnection.processRequest(ServerRpcConnection.java:564) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.ServerRpcConnection.processOneRpc(ServerRpcConnection.java:364) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyServerRpcConnection.process(NettyServerRpcConnection.java:89) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcServerRequestDecoder.channelRead0(NettyRpcServerRequestDecoder.java:56) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcServerRequestDecoder.channelRead0(NettyRpcServerRequestDecoder.java:31) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollStreamChannel$EpollStreamUnsafe.epollInReady(AbstractEpollStreamChannel.java:799) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:501) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T14:31:26,458 DEBUG [RPCClient-NioEventLoopGroup-6-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1732631484097.40329a66ba4e990f8463ecffc5dfdaa9., hostname=19945b63c2b4,35419,1732631469384, seqNum=12 , the old value is region=testReplayEditsAfterRegionMovedWithMultiCF,,1732631484097.40329a66ba4e990f8463ecffc5dfdaa9., hostname=19945b63c2b4,35419,1732631469384, seqNum=12, error=org.apache.hadoop.hbase.regionserver.RegionServerAbortedException: Server 19945b63c2b4,35419,1732631469384 aborting 2024-11-26T14:31:26,458 DEBUG [RPCClient-NioEventLoopGroup-6-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1732631484097.40329a66ba4e990f8463ecffc5dfdaa9., hostname=19945b63c2b4,35419,1732631469384, seqNum=12 is org.apache.hadoop.hbase.regionserver.RegionServerAbortedException: Server 19945b63c2b4,35419,1732631469384 aborting 2024-11-26T14:31:26,458 DEBUG [RPCClient-NioEventLoopGroup-6-5 {}] client.AsyncRegionLocatorHelper(88): Try removing region=testReplayEditsAfterRegionMovedWithMultiCF,,1732631484097.40329a66ba4e990f8463ecffc5dfdaa9., hostname=19945b63c2b4,35419,1732631469384, seqNum=12 from cache 2024-11-26T14:31:26,549 INFO [RS:0;19945b63c2b4:35419 {}] regionserver.HRegionServer(976): stopping server 19945b63c2b4,35419,1732631469384; all regions closed. 2024-11-26T14:31:26,554 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36307 is added to blk_1073741835_1011 (size=1677) 2024-11-26T14:31:26,554 WARN [Close-WAL-Writer-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(650): complete file /user/jenkins/test-data/2de93217-0593-bca0-5c5f-879b6362c76f/WALs/19945b63c2b4,35419,1732631469384/19945b63c2b4%2C35419%2C1732631469384.1732631471132 not finished, retry = 0 2024-11-26T14:31:26,554 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38147 is added to blk_1073741835_1011 (size=1677) 2024-11-26T14:31:26,555 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36277 is added to blk_1073741835_1011 (size=1677) 2024-11-26T14:31:26,656 DEBUG [RS:0;19945b63c2b4:35419 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T14:31:26,656 INFO [RS:0;19945b63c2b4:35419 {}] regionserver.LeaseManager(133): Closed leases 2024-11-26T14:31:26,656 INFO [RS:0;19945b63c2b4:35419 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-26T14:31:26,656 INFO [RS:0;19945b63c2b4:35419 {}] hbase.ChoreService(370): Chore service for: regionserver/19945b63c2b4:0 had [ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-11-26T14:31:26,656 INFO [regionserver/19945b63c2b4:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-26T14:31:26,656 INFO [RS:0;19945b63c2b4:35419 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-26T14:31:26,656 INFO [RS:0;19945b63c2b4:35419 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-26T14:31:26,656 INFO [RS:0;19945b63c2b4:35419 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-26T14:31:26,656 INFO [RS:0;19945b63c2b4:35419 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-26T14:31:26,657 INFO [RS:0;19945b63c2b4:35419 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.3:35419 2024-11-26T14:31:26,668 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testReplayEditsAfterRegionMovedWithMultiCF', row='r1', locateType=CURRENT is [region=testReplayEditsAfterRegionMovedWithMultiCF,,1732631484097.40329a66ba4e990f8463ecffc5dfdaa9., hostname=19945b63c2b4,35419,1732631469384, seqNum=18] 2024-11-26T14:31:26,670 WARN [RPCClient-NioEventLoopGroup-6-5 {}] ipc.NettyRpcConnection$2(409): Exception encountered while connecting to the server 19945b63c2b4:35419 org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: Connection refused: 19945b63c2b4/172.17.0.3:35419 Caused by: java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hbase.thirdparty.io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(NioSocketChannel.java:336) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:339) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T14:31:26,671 DEBUG [RPCClient-NioEventLoopGroup-6-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1732631484097.40329a66ba4e990f8463ecffc5dfdaa9., hostname=19945b63c2b4,35419,1732631469384, seqNum=18 , the old value is region=testReplayEditsAfterRegionMovedWithMultiCF,,1732631484097.40329a66ba4e990f8463ecffc5dfdaa9., hostname=19945b63c2b4,35419,1732631469384, seqNum=18, error=java.net.ConnectException: Call to address=19945b63c2b4:35419 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: Connection refused: 19945b63c2b4/172.17.0.3:35419 2024-11-26T14:31:26,671 DEBUG [RPCClient-NioEventLoopGroup-6-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1732631484097.40329a66ba4e990f8463ecffc5dfdaa9., hostname=19945b63c2b4,35419,1732631469384, seqNum=18 is java.net.ConnectException: Connection refused 2024-11-26T14:31:26,671 DEBUG [RPCClient-NioEventLoopGroup-6-5 {}] client.AsyncRegionLocatorHelper(88): Try removing region=testReplayEditsAfterRegionMovedWithMultiCF,,1732631484097.40329a66ba4e990f8463ecffc5dfdaa9., hostname=19945b63c2b4,35419,1732631469384, seqNum=18 from cache 2024-11-26T14:31:26,671 DEBUG [RPCClient-NioEventLoopGroup-6-5 {}] ipc.FailedServers(52): Added failed server with address 19945b63c2b4:35419 to list caused by org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: Connection refused: 19945b63c2b4/172.17.0.3:35419 2024-11-26T14:31:26,731 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35419-0x101778e13450001, quorum=127.0.0.1:49403, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/19945b63c2b4,35419,1732631469384 2024-11-26T14:31:26,731 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43577-0x101778e13450000, quorum=127.0.0.1:49403, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-26T14:31:26,731 INFO [RS:0;19945b63c2b4:35419 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-26T14:31:26,746 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [19945b63c2b4,35419,1732631469384] 2024-11-26T14:31:26,755 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/19945b63c2b4,35419,1732631469384 already deleted, retry=false 2024-11-26T14:31:26,755 INFO [RegionServerTracker-0 {}] master.ServerManager(695): Processing expiration of 19945b63c2b4,35419,1732631469384 on 19945b63c2b4,43577,1732631468672 2024-11-26T14:31:26,761 DEBUG [RegionServerTracker-0 {}] procedure2.ProcedureExecutor(1139): Stored pid=13, state=RUNNABLE:SERVER_CRASH_START, hasLock=false; ServerCrashProcedure 19945b63c2b4,35419,1732631469384, splitWal=true, meta=false 2024-11-26T14:31:26,764 INFO [RegionServerTracker-0 {}] assignment.AssignmentManager(1999): Scheduled ServerCrashProcedure pid=13 for 19945b63c2b4,35419,1732631469384 (carryingMeta=false) 19945b63c2b4,35419,1732631469384/CRASHED/regionCount=1/lock=java.util.concurrent.locks.ReentrantReadWriteLock@59e6289d[Write locks = 1, Read locks = 0], oldState=ONLINE. 2024-11-26T14:31:26,764 INFO [PEWorker-3 {}] procedure.ServerCrashProcedure(169): Start pid=13, state=RUNNABLE:SERVER_CRASH_START, hasLock=true; ServerCrashProcedure 19945b63c2b4,35419,1732631469384, splitWal=true, meta=false 2024-11-26T14:31:26,766 INFO [PEWorker-3 {}] procedure.ServerCrashProcedure(207): 19945b63c2b4,35419,1732631469384 had 1 regions 2024-11-26T14:31:26,767 INFO [PEWorker-3 {}] procedure.ServerCrashProcedure(339): Splitting WALs pid=13, state=RUNNABLE:SERVER_CRASH_SPLIT_LOGS, hasLock=true; ServerCrashProcedure 19945b63c2b4,35419,1732631469384, splitWal=true, meta=false, isMeta: false 2024-11-26T14:31:26,769 DEBUG [PEWorker-3 {}] master.MasterWalManager(329): Renamed region directory: hdfs://localhost:44999/user/jenkins/test-data/2de93217-0593-bca0-5c5f-879b6362c76f/WALs/19945b63c2b4,35419,1732631469384-splitting 2024-11-26T14:31:26,770 INFO [PEWorker-3 {}] master.SplitWALManager(105): 19945b63c2b4,35419,1732631469384 WAL count=1, meta=false 2024-11-26T14:31:26,772 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=14, ppid=13, state=RUNNABLE:ACQUIRE_SPLIT_WAL_WORKER, hasLock=false; SplitWALProcedure 19945b63c2b4%2C35419%2C1732631469384.1732631471132}] 2024-11-26T14:31:26,777 DEBUG [PEWorker-1 {}] master.SplitWALManager(158): Acquired split WAL worker=19945b63c2b4,40179,1732631469496 2024-11-26T14:31:26,779 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=15, ppid=14, state=RUNNABLE, hasLock=false; SplitWALRemoteProcedure 19945b63c2b4%2C35419%2C1732631469384.1732631471132, worker=19945b63c2b4,40179,1732631469496}] 2024-11-26T14:31:26,845 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35419-0x101778e13450001, quorum=127.0.0.1:49403, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-26T14:31:26,845 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35419-0x101778e13450001, quorum=127.0.0.1:49403, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-26T14:31:26,846 INFO [RS:0;19945b63c2b4:35419 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-26T14:31:26,846 INFO [RS:0;19945b63c2b4:35419 {}] regionserver.HRegionServer(1031): Exiting; stopping=19945b63c2b4,35419,1732631469384; zookeeper connection closed. 2024-11-26T14:31:26,846 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@68520f5f {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@68520f5f 2024-11-26T14:31:26,945 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40179 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SplitWALCallable, pid=15 2024-11-26T14:31:26,965 INFO [RS_LOG_REPLAY_OPS-regionserver/19945b63c2b4:0-0 {event_type=RS_LOG_REPLAY, pid=15}] wal.WALSplitter(299): Splitting hdfs://localhost:44999/user/jenkins/test-data/2de93217-0593-bca0-5c5f-879b6362c76f/WALs/19945b63c2b4,35419,1732631469384-splitting/19945b63c2b4%2C35419%2C1732631469384.1732631471132, size=1.6 K (1677bytes) 2024-11-26T14:31:26,965 INFO [RS_LOG_REPLAY_OPS-regionserver/19945b63c2b4:0-0 {event_type=RS_LOG_REPLAY, pid=15}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:44999/user/jenkins/test-data/2de93217-0593-bca0-5c5f-879b6362c76f/WALs/19945b63c2b4,35419,1732631469384-splitting/19945b63c2b4%2C35419%2C1732631469384.1732631471132 2024-11-26T14:31:26,966 INFO [RS_LOG_REPLAY_OPS-regionserver/19945b63c2b4:0-0 {event_type=RS_LOG_REPLAY, pid=15}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:44999/user/jenkins/test-data/2de93217-0593-bca0-5c5f-879b6362c76f/WALs/19945b63c2b4,35419,1732631469384-splitting/19945b63c2b4%2C35419%2C1732631469384.1732631471132 after 1ms 2024-11-26T14:31:26,969 DEBUG [RS_LOG_REPLAY_OPS-regionserver/19945b63c2b4:0-0 {event_type=RS_LOG_REPLAY, pid=15}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:44999/user/jenkins/test-data/2de93217-0593-bca0-5c5f-879b6362c76f/WALs/19945b63c2b4,35419,1732631469384-splitting/19945b63c2b4%2C35419%2C1732631469384.1732631471132: isRecoveredEdits=false, hasTagCompression=true, hasValueCompression=false, valueCompressionType=null 2024-11-26T14:31:26,969 INFO [RS_LOG_REPLAY_OPS-regionserver/19945b63c2b4:0-0 {event_type=RS_LOG_REPLAY, pid=15}] wal.WALSplitter(310): Open hdfs://localhost:44999/user/jenkins/test-data/2de93217-0593-bca0-5c5f-879b6362c76f/WALs/19945b63c2b4,35419,1732631469384-splitting/19945b63c2b4%2C35419%2C1732631469384.1732631471132 took 4ms 2024-11-26T14:31:26,976 DEBUG [RS_LOG_REPLAY_OPS-regionserver/19945b63c2b4:0-0 {event_type=RS_LOG_REPLAY, pid=15}] wal.WALSplitter(352): Last flushed sequenceid for 40329a66ba4e990f8463ecffc5dfdaa9: last_flushed_sequence_id: 12 store_sequence_id { family_name: "cf1" sequence_id: 12 } store_sequence_id { family_name: "cf2" sequence_id: 12 } 2024-11-26T14:31:26,977 DEBUG [RS_LOG_REPLAY_OPS-regionserver/19945b63c2b4:0-0 {event_type=RS_LOG_REPLAY, pid=15}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:44999/user/jenkins/test-data/2de93217-0593-bca0-5c5f-879b6362c76f/WALs/19945b63c2b4,35419,1732631469384-splitting/19945b63c2b4%2C35419%2C1732631469384.1732631471132 so closing down 2024-11-26T14:31:26,977 DEBUG [RS_LOG_REPLAY_OPS-regionserver/19945b63c2b4:0-0 {event_type=RS_LOG_REPLAY, pid=15}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-26T14:31:26,977 INFO [RS_LOG_REPLAY_OPS-regionserver/19945b63c2b4:0-0 {event_type=RS_LOG_REPLAY, pid=15}] wal.OutputSink(145): 3 split writer threads finished 2024-11-26T14:31:26,977 INFO [RS_LOG_REPLAY_OPS-regionserver/19945b63c2b4:0-0 {event_type=RS_LOG_REPLAY, pid=15}] wal.WALSplitter(425): Processed 6 edits across 0 Regions in 7 ms; skipped=6; WAL=hdfs://localhost:44999/user/jenkins/test-data/2de93217-0593-bca0-5c5f-879b6362c76f/WALs/19945b63c2b4,35419,1732631469384-splitting/19945b63c2b4%2C35419%2C1732631469384.1732631471132, size=1.6 K, length=1677, corrupted=false, cancelled=false 2024-11-26T14:31:26,977 DEBUG [RS_LOG_REPLAY_OPS-regionserver/19945b63c2b4:0-0 {event_type=RS_LOG_REPLAY, pid=15}] wal.WALSplitter(428): Completed split of hdfs://localhost:44999/user/jenkins/test-data/2de93217-0593-bca0-5c5f-879b6362c76f/WALs/19945b63c2b4,35419,1732631469384-splitting/19945b63c2b4%2C35419%2C1732631469384.1732631471132, journal: Splitting hdfs://localhost:44999/user/jenkins/test-data/2de93217-0593-bca0-5c5f-879b6362c76f/WALs/19945b63c2b4,35419,1732631469384-splitting/19945b63c2b4%2C35419%2C1732631469384.1732631471132, size=1.6 K (1677bytes) at 1732631486965Finishing writing output for hdfs://localhost:44999/user/jenkins/test-data/2de93217-0593-bca0-5c5f-879b6362c76f/WALs/19945b63c2b4,35419,1732631469384-splitting/19945b63c2b4%2C35419%2C1732631469384.1732631471132 so closing down at 1732631486977 (+12 ms)3 split writer threads finished at 1732631486977Processed 6 edits across 0 Regions in 7 ms; skipped=6; WAL=hdfs://localhost:44999/user/jenkins/test-data/2de93217-0593-bca0-5c5f-879b6362c76f/WALs/19945b63c2b4,35419,1732631469384-splitting/19945b63c2b4%2C35419%2C1732631469384.1732631471132, size=1.6 K, length=1677, corrupted=false, cancelled=false at 1732631486977 2024-11-26T14:31:26,977 DEBUG [RS_LOG_REPLAY_OPS-regionserver/19945b63c2b4:0-0 {event_type=RS_LOG_REPLAY, pid=15}] regionserver.SplitLogWorker(218): Done splitting WAL hdfs://localhost:44999/user/jenkins/test-data/2de93217-0593-bca0-5c5f-879b6362c76f/WALs/19945b63c2b4,35419,1732631469384-splitting/19945b63c2b4%2C35419%2C1732631469384.1732631471132 2024-11-26T14:31:26,979 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testReplayEditsAfterRegionMovedWithMultiCF', row='r1', locateType=CURRENT is [region=testReplayEditsAfterRegionMovedWithMultiCF,,1732631484097.40329a66ba4e990f8463ecffc5dfdaa9., hostname=19945b63c2b4,35419,1732631469384, seqNum=18] 2024-11-26T14:31:26,979 DEBUG [RS_LOG_REPLAY_OPS-regionserver/19945b63c2b4:0-0 {event_type=RS_LOG_REPLAY, pid=15}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=15 2024-11-26T14:31:26,979 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] ipc.AbstractRpcClient(357): Not trying to connect to 19945b63c2b4:35419 this server is in the failed servers list 2024-11-26T14:31:26,980 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1732631484097.40329a66ba4e990f8463ecffc5dfdaa9., hostname=19945b63c2b4,35419,1732631469384, seqNum=18 , the old value is region=testReplayEditsAfterRegionMovedWithMultiCF,,1732631484097.40329a66ba4e990f8463ecffc5dfdaa9., hostname=19945b63c2b4,35419,1732631469384, seqNum=18, error=org.apache.hadoop.hbase.ipc.FailedServerException: Call to address=19945b63c2b4:35419 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: 19945b63c2b4:35419 2024-11-26T14:31:26,980 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1732631484097.40329a66ba4e990f8463ecffc5dfdaa9., hostname=19945b63c2b4,35419,1732631469384, seqNum=18 is org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: 19945b63c2b4:35419 2024-11-26T14:31:26,980 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncRegionLocatorHelper(88): Try removing region=testReplayEditsAfterRegionMovedWithMultiCF,,1732631484097.40329a66ba4e990f8463ecffc5dfdaa9., hostname=19945b63c2b4,35419,1732631469384, seqNum=18 from cache 2024-11-26T14:31:26,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43577 {}] master.HMaster(4169): Remote procedure done, pid=15 2024-11-26T14:31:26,986 INFO [PEWorker-4 {}] wal.WALSplitUtil(143): Moved hdfs://localhost:44999/user/jenkins/test-data/2de93217-0593-bca0-5c5f-879b6362c76f/WALs/19945b63c2b4,35419,1732631469384-splitting/19945b63c2b4%2C35419%2C1732631469384.1732631471132 to hdfs://localhost:44999/user/jenkins/test-data/2de93217-0593-bca0-5c5f-879b6362c76f/oldWALs 2024-11-26T14:31:26,989 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=15, resume processing ppid=14 2024-11-26T14:31:26,989 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=15, ppid=14, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 19945b63c2b4%2C35419%2C1732631469384.1732631471132, worker=19945b63c2b4,40179,1732631469496 in 207 msec 2024-11-26T14:31:26,990 DEBUG [PEWorker-5 {}] master.SplitWALManager(172): Release split WAL worker=19945b63c2b4,40179,1732631469496 2024-11-26T14:31:26,993 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=14, resume processing ppid=13 2024-11-26T14:31:26,993 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=14, ppid=13, state=SUCCESS, hasLock=false; SplitWALProcedure 19945b63c2b4%2C35419%2C1732631469384.1732631471132, worker=19945b63c2b4,40179,1732631469496 in 220 msec 2024-11-26T14:31:26,995 INFO [PEWorker-3 {}] master.SplitLogManager(171): hdfs://localhost:44999/user/jenkins/test-data/2de93217-0593-bca0-5c5f-879b6362c76f/WALs/19945b63c2b4,35419,1732631469384-splitting dir is empty, no logs to split. 2024-11-26T14:31:26,995 INFO [PEWorker-3 {}] master.SplitWALManager(105): 19945b63c2b4,35419,1732631469384 WAL count=0, meta=false 2024-11-26T14:31:26,995 DEBUG [PEWorker-3 {}] procedure.ServerCrashProcedure(329): Check if 19945b63c2b4,35419,1732631469384 WAL splitting is done? wals=0, meta=false 2024-11-26T14:31:26,997 INFO [PEWorker-3 {}] procedure.ServerCrashProcedure(321): Remove WAL directory for 19945b63c2b4,35419,1732631469384 failed, ignore...File hdfs://localhost:44999/user/jenkins/test-data/2de93217-0593-bca0-5c5f-879b6362c76f/WALs/19945b63c2b4,35419,1732631469384-splitting does not exist. 2024-11-26T14:31:27,000 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=16, ppid=13, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=40329a66ba4e990f8463ecffc5dfdaa9, ASSIGN}] 2024-11-26T14:31:27,001 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=16, ppid=13, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=40329a66ba4e990f8463ecffc5dfdaa9, ASSIGN 2024-11-26T14:31:27,003 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=16, ppid=13, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=40329a66ba4e990f8463ecffc5dfdaa9, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-11-26T14:31:27,153 DEBUG [19945b63c2b4:43577 {}] balancer.BalancerClusterState(204): Hosts are {19945b63c2b4=0} racks are {/default-rack=0} 2024-11-26T14:31:27,154 DEBUG [19945b63c2b4:43577 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-11-26T14:31:27,154 DEBUG [19945b63c2b4:43577 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-11-26T14:31:27,154 DEBUG [19945b63c2b4:43577 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-11-26T14:31:27,154 DEBUG [19945b63c2b4:43577 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-11-26T14:31:27,154 INFO [19945b63c2b4:43577 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-11-26T14:31:27,154 INFO [19945b63c2b4:43577 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-11-26T14:31:27,154 DEBUG [19945b63c2b4:43577 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-11-26T14:31:27,154 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=16 updating hbase:meta row=40329a66ba4e990f8463ecffc5dfdaa9, regionState=OPENING, regionLocation=19945b63c2b4,37467,1732631469538 2024-11-26T14:31:27,159 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=16, ppid=13, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=40329a66ba4e990f8463ecffc5dfdaa9, ASSIGN because future has completed 2024-11-26T14:31:27,159 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=17, ppid=16, state=RUNNABLE, hasLock=false; OpenRegionProcedure 40329a66ba4e990f8463ecffc5dfdaa9, server=19945b63c2b4,37467,1732631469538}] 2024-11-26T14:31:27,326 INFO [RS_OPEN_REGION-regionserver/19945b63c2b4:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] handler.AssignRegionHandler(132): Open testReplayEditsAfterRegionMovedWithMultiCF,,1732631484097.40329a66ba4e990f8463ecffc5dfdaa9. 2024-11-26T14:31:27,327 DEBUG [RS_OPEN_REGION-regionserver/19945b63c2b4:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegion(7752): Opening region: {ENCODED => 40329a66ba4e990f8463ecffc5dfdaa9, NAME => 'testReplayEditsAfterRegionMovedWithMultiCF,,1732631484097.40329a66ba4e990f8463ecffc5dfdaa9.', STARTKEY => '', ENDKEY => ''} 2024-11-26T14:31:27,328 DEBUG [RS_OPEN_REGION-regionserver/19945b63c2b4:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testReplayEditsAfterRegionMovedWithMultiCF 40329a66ba4e990f8463ecffc5dfdaa9 2024-11-26T14:31:27,328 DEBUG [RS_OPEN_REGION-regionserver/19945b63c2b4:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegion(898): Instantiated testReplayEditsAfterRegionMovedWithMultiCF,,1732631484097.40329a66ba4e990f8463ecffc5dfdaa9.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-26T14:31:27,328 DEBUG [RS_OPEN_REGION-regionserver/19945b63c2b4:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegion(7794): checking encryption for 40329a66ba4e990f8463ecffc5dfdaa9 2024-11-26T14:31:27,328 DEBUG [RS_OPEN_REGION-regionserver/19945b63c2b4:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegion(7797): checking classloading for 40329a66ba4e990f8463ecffc5dfdaa9 2024-11-26T14:31:27,330 INFO [StoreOpener-40329a66ba4e990f8463ecffc5dfdaa9-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf1 of region 40329a66ba4e990f8463ecffc5dfdaa9 2024-11-26T14:31:27,331 INFO [StoreOpener-40329a66ba4e990f8463ecffc5dfdaa9-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 40329a66ba4e990f8463ecffc5dfdaa9 columnFamilyName cf1 2024-11-26T14:31:27,331 DEBUG [StoreOpener-40329a66ba4e990f8463ecffc5dfdaa9-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:31:27,339 DEBUG [StoreOpener-40329a66ba4e990f8463ecffc5dfdaa9-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:44999/user/jenkins/test-data/2de93217-0593-bca0-5c5f-879b6362c76f/data/default/testReplayEditsAfterRegionMovedWithMultiCF/40329a66ba4e990f8463ecffc5dfdaa9/cf1/996eb589593143149c38b54e30a0fb3f 2024-11-26T14:31:27,339 INFO [StoreOpener-40329a66ba4e990f8463ecffc5dfdaa9-1 {}] regionserver.HStore(327): Store=40329a66ba4e990f8463ecffc5dfdaa9/cf1, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-26T14:31:27,339 INFO [StoreOpener-40329a66ba4e990f8463ecffc5dfdaa9-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf2 of region 40329a66ba4e990f8463ecffc5dfdaa9 2024-11-26T14:31:27,340 INFO [StoreOpener-40329a66ba4e990f8463ecffc5dfdaa9-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 40329a66ba4e990f8463ecffc5dfdaa9 columnFamilyName cf2 2024-11-26T14:31:27,340 DEBUG [StoreOpener-40329a66ba4e990f8463ecffc5dfdaa9-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:31:27,347 DEBUG [StoreOpener-40329a66ba4e990f8463ecffc5dfdaa9-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:44999/user/jenkins/test-data/2de93217-0593-bca0-5c5f-879b6362c76f/data/default/testReplayEditsAfterRegionMovedWithMultiCF/40329a66ba4e990f8463ecffc5dfdaa9/cf2/aad6938c9a964b5d92fe3112370fd466 2024-11-26T14:31:27,348 INFO [StoreOpener-40329a66ba4e990f8463ecffc5dfdaa9-1 {}] regionserver.HStore(327): Store=40329a66ba4e990f8463ecffc5dfdaa9/cf2, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-26T14:31:27,348 DEBUG [RS_OPEN_REGION-regionserver/19945b63c2b4:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegion(1038): replaying wal for 40329a66ba4e990f8463ecffc5dfdaa9 2024-11-26T14:31:27,349 DEBUG [RS_OPEN_REGION-regionserver/19945b63c2b4:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44999/user/jenkins/test-data/2de93217-0593-bca0-5c5f-879b6362c76f/data/default/testReplayEditsAfterRegionMovedWithMultiCF/40329a66ba4e990f8463ecffc5dfdaa9 2024-11-26T14:31:27,350 DEBUG [RS_OPEN_REGION-regionserver/19945b63c2b4:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44999/user/jenkins/test-data/2de93217-0593-bca0-5c5f-879b6362c76f/data/default/testReplayEditsAfterRegionMovedWithMultiCF/40329a66ba4e990f8463ecffc5dfdaa9 2024-11-26T14:31:27,351 DEBUG [RS_OPEN_REGION-regionserver/19945b63c2b4:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegion(1048): stopping wal replay for 40329a66ba4e990f8463ecffc5dfdaa9 2024-11-26T14:31:27,351 DEBUG [RS_OPEN_REGION-regionserver/19945b63c2b4:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegion(1060): Cleaning up temporary data for 40329a66ba4e990f8463ecffc5dfdaa9 2024-11-26T14:31:27,351 DEBUG [RS_OPEN_REGION-regionserver/19945b63c2b4:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsAfterRegionMovedWithMultiCF descriptor;using region.getMemStoreFlushHeapSize/# of families (64.0 M)) instead. 2024-11-26T14:31:27,353 DEBUG [RS_OPEN_REGION-regionserver/19945b63c2b4:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegion(1093): writing seq id for 40329a66ba4e990f8463ecffc5dfdaa9 2024-11-26T14:31:27,354 INFO [RS_OPEN_REGION-regionserver/19945b63c2b4:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegion(1114): Opened 40329a66ba4e990f8463ecffc5dfdaa9; next sequenceid=18; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=70359811, jitterRate=0.04844288527965546}}}, FlushLargeStoresPolicy{flushSizeLowerBound=67108864} 2024-11-26T14:31:27,354 DEBUG [RS_OPEN_REGION-regionserver/19945b63c2b4:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 40329a66ba4e990f8463ecffc5dfdaa9 2024-11-26T14:31:27,354 DEBUG [RS_OPEN_REGION-regionserver/19945b63c2b4:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegion(1006): Region open journal for 40329a66ba4e990f8463ecffc5dfdaa9: Running coprocessor pre-open hook at 1732631487328Writing region info on filesystem at 1732631487328Initializing all the Stores at 1732631487329 (+1 ms)Instantiating store for column family {NAME => 'cf1', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732631487329Instantiating store for column family {NAME => 'cf2', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732631487329Cleaning up temporary data from old regions at 1732631487351 (+22 ms)Running coprocessor post-open hooks at 1732631487354 (+3 ms)Region opened successfully at 1732631487354 2024-11-26T14:31:27,356 INFO [RS_OPEN_REGION-regionserver/19945b63c2b4:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegionServer(2236): Post open deploy tasks for testReplayEditsAfterRegionMovedWithMultiCF,,1732631484097.40329a66ba4e990f8463ecffc5dfdaa9., pid=17, masterSystemTime=1732631487314 2024-11-26T14:31:27,359 DEBUG [RS_OPEN_REGION-regionserver/19945b63c2b4:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegionServer(2266): Finished post open deploy task for testReplayEditsAfterRegionMovedWithMultiCF,,1732631484097.40329a66ba4e990f8463ecffc5dfdaa9. 2024-11-26T14:31:27,359 INFO [RS_OPEN_REGION-regionserver/19945b63c2b4:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] handler.AssignRegionHandler(153): Opened testReplayEditsAfterRegionMovedWithMultiCF,,1732631484097.40329a66ba4e990f8463ecffc5dfdaa9. 2024-11-26T14:31:27,359 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=16 updating hbase:meta row=40329a66ba4e990f8463ecffc5dfdaa9, regionState=OPEN, openSeqNum=18, regionLocation=19945b63c2b4,37467,1732631469538 2024-11-26T14:31:27,362 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=17, ppid=16, state=RUNNABLE, hasLock=false; OpenRegionProcedure 40329a66ba4e990f8463ecffc5dfdaa9, server=19945b63c2b4,37467,1732631469538 because future has completed 2024-11-26T14:31:27,366 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=17, resume processing ppid=16 2024-11-26T14:31:27,366 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=17, ppid=16, state=SUCCESS, hasLock=false; OpenRegionProcedure 40329a66ba4e990f8463ecffc5dfdaa9, server=19945b63c2b4,37467,1732631469538 in 204 msec 2024-11-26T14:31:27,368 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=16, resume processing ppid=13 2024-11-26T14:31:27,368 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=16, ppid=13, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=40329a66ba4e990f8463ecffc5dfdaa9, ASSIGN in 366 msec 2024-11-26T14:31:27,368 INFO [PEWorker-4 {}] procedure.ServerCrashProcedure(291): removed crashed server 19945b63c2b4,35419,1732631469384 after splitting done 2024-11-26T14:31:27,371 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=13, state=SUCCESS, hasLock=false; ServerCrashProcedure 19945b63c2b4,35419,1732631469384, splitWal=true, meta=false in 613 msec 2024-11-26T14:31:27,489 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testReplayEditsAfterRegionMovedWithMultiCF', row='r1', locateType=CURRENT is [region=testReplayEditsAfterRegionMovedWithMultiCF,,1732631484097.40329a66ba4e990f8463ecffc5dfdaa9., hostname=19945b63c2b4,37467,1732631469538, seqNum=18] 2024-11-26T14:31:27,503 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestAsyncWALReplayCompressed#testReplayEditsAfterRegionMovedWithMultiCF Thread=415 (was 416), OpenFileDescriptor=1051 (was 1023) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=248 (was 248), ProcessCount=11 (was 11), AvailableMemoryMB=9058 (was 9088) 2024-11-26T14:31:27,505 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1051 is superior to 1024 2024-11-26T14:31:27,516 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestAsyncWALReplayCompressed#testReplayEditsAfterPartialFlush Thread=415, OpenFileDescriptor=1051, MaxFileDescriptor=1048576, SystemLoadAverage=248, ProcessCount=11, AvailableMemoryMB=9058 2024-11-26T14:31:27,516 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1051 is superior to 1024 2024-11-26T14:31:27,530 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-26T14:31:27,532 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-26T14:31:27,532 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-26T14:31:27,535 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-53119873, suffix=, logDir=hdfs://localhost:44999/hbase/WALs/hregion-53119873, archiveDir=hdfs://localhost:44999/hbase/oldWALs, maxLogs=32 2024-11-26T14:31:27,546 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-53119873/hregion-53119873.1732631487535, exclude list is [], retry=0 2024-11-26T14:31:27,549 DEBUG [AsyncFSWAL-20-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38147,DS-44b12d44-38d0-4c76-b56f-3ca8d8caa398,DISK] 2024-11-26T14:31:27,550 DEBUG [AsyncFSWAL-20-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36277,DS-9cb9383d-a6c8-4ba6-83b3-6706331a6539,DISK] 2024-11-26T14:31:27,550 DEBUG [AsyncFSWAL-20-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36307,DS-dd37de78-b3d5-4d7c-8223-4b15101c21bf,DISK] 2024-11-26T14:31:27,552 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-53119873/hregion-53119873.1732631487535 2024-11-26T14:31:27,552 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:35081:35081),(127.0.0.1/127.0.0.1:40449:40449),(127.0.0.1/127.0.0.1:34431:34431)] 2024-11-26T14:31:27,552 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => 5b9f51ef77304bf0438669929febadea, NAME => 'testReplayEditsWrittenViaHRegion,,1732631487530.5b9f51ef77304bf0438669929febadea.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testReplayEditsWrittenViaHRegion', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:44999/hbase 2024-11-26T14:31:27,562 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36277 is added to blk_1073741893_1071 (size=67) 2024-11-26T14:31:27,562 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38147 is added to blk_1073741893_1071 (size=67) 2024-11-26T14:31:27,562 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36307 is added to blk_1073741893_1071 (size=67) 2024-11-26T14:31:27,563 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenViaHRegion,,1732631487530.5b9f51ef77304bf0438669929febadea.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-26T14:31:27,564 INFO [StoreOpener-5b9f51ef77304bf0438669929febadea-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 5b9f51ef77304bf0438669929febadea 2024-11-26T14:31:27,565 INFO [StoreOpener-5b9f51ef77304bf0438669929febadea-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 5b9f51ef77304bf0438669929febadea columnFamilyName a 2024-11-26T14:31:27,566 DEBUG [StoreOpener-5b9f51ef77304bf0438669929febadea-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:31:27,566 INFO [StoreOpener-5b9f51ef77304bf0438669929febadea-1 {}] regionserver.HStore(327): Store=5b9f51ef77304bf0438669929febadea/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-26T14:31:27,566 INFO [StoreOpener-5b9f51ef77304bf0438669929febadea-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 5b9f51ef77304bf0438669929febadea 2024-11-26T14:31:27,567 INFO [StoreOpener-5b9f51ef77304bf0438669929febadea-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 5b9f51ef77304bf0438669929febadea columnFamilyName b 2024-11-26T14:31:27,567 DEBUG [StoreOpener-5b9f51ef77304bf0438669929febadea-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:31:27,568 INFO [StoreOpener-5b9f51ef77304bf0438669929febadea-1 {}] regionserver.HStore(327): Store=5b9f51ef77304bf0438669929febadea/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-26T14:31:27,568 INFO [StoreOpener-5b9f51ef77304bf0438669929febadea-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 5b9f51ef77304bf0438669929febadea 2024-11-26T14:31:27,569 INFO [StoreOpener-5b9f51ef77304bf0438669929febadea-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 5b9f51ef77304bf0438669929febadea columnFamilyName c 2024-11-26T14:31:27,569 DEBUG [StoreOpener-5b9f51ef77304bf0438669929febadea-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:31:27,570 INFO [StoreOpener-5b9f51ef77304bf0438669929febadea-1 {}] regionserver.HStore(327): Store=5b9f51ef77304bf0438669929febadea/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-26T14:31:27,570 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 5b9f51ef77304bf0438669929febadea 2024-11-26T14:31:27,571 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44999/hbase/data/default/testReplayEditsWrittenViaHRegion/5b9f51ef77304bf0438669929febadea 2024-11-26T14:31:27,572 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44999/hbase/data/default/testReplayEditsWrittenViaHRegion/5b9f51ef77304bf0438669929febadea 2024-11-26T14:31:27,573 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 5b9f51ef77304bf0438669929febadea 2024-11-26T14:31:27,573 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 5b9f51ef77304bf0438669929febadea 2024-11-26T14:31:27,573 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsWrittenViaHRegion descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-26T14:31:27,574 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 5b9f51ef77304bf0438669929febadea 2024-11-26T14:31:27,577 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44999/hbase/data/default/testReplayEditsWrittenViaHRegion/5b9f51ef77304bf0438669929febadea/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-26T14:31:27,577 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 5b9f51ef77304bf0438669929febadea; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=60179879, jitterRate=-0.1032499223947525}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-26T14:31:27,578 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 5b9f51ef77304bf0438669929febadea: Writing region info on filesystem at 1732631487563Initializing all the Stores at 1732631487564 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732631487564Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732631487564Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732631487564Cleaning up temporary data from old regions at 1732631487573 (+9 ms)Region opened successfully at 1732631487578 (+5 ms) 2024-11-26T14:31:27,578 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 5b9f51ef77304bf0438669929febadea, disabling compactions & flushes 2024-11-26T14:31:27,578 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsWrittenViaHRegion,,1732631487530.5b9f51ef77304bf0438669929febadea. 2024-11-26T14:31:27,578 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsWrittenViaHRegion,,1732631487530.5b9f51ef77304bf0438669929febadea. 2024-11-26T14:31:27,578 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsWrittenViaHRegion,,1732631487530.5b9f51ef77304bf0438669929febadea. after waiting 0 ms 2024-11-26T14:31:27,578 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsWrittenViaHRegion,,1732631487530.5b9f51ef77304bf0438669929febadea. 2024-11-26T14:31:27,578 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsWrittenViaHRegion,,1732631487530.5b9f51ef77304bf0438669929febadea. 2024-11-26T14:31:27,578 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 5b9f51ef77304bf0438669929febadea: Waiting for close lock at 1732631487578Disabling compacts and flushes for region at 1732631487578Disabling writes for close at 1732631487578Writing region close event to WAL at 1732631487578Closed at 1732631487578 2024-11-26T14:31:27,581 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36307 is added to blk_1073741892_1070 (size=93) 2024-11-26T14:31:27,581 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36277 is added to blk_1073741892_1070 (size=93) 2024-11-26T14:31:27,581 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38147 is added to blk_1073741892_1070 (size=93) 2024-11-26T14:31:27,584 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-11-26T14:31:27,584 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-53119873:(num 1732631487535) 2024-11-26T14:31:27,584 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-26T14:31:27,586 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:44999/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1732631487529, archiveDir=hdfs://localhost:44999/hbase/oldWALs, maxLogs=32 2024-11-26T14:31:27,598 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1732631487529/wal.1732631487586, exclude list is [], retry=0 2024-11-26T14:31:27,601 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36307,DS-dd37de78-b3d5-4d7c-8223-4b15101c21bf,DISK] 2024-11-26T14:31:27,602 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36277,DS-9cb9383d-a6c8-4ba6-83b3-6706331a6539,DISK] 2024-11-26T14:31:27,602 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38147,DS-44b12d44-38d0-4c76-b56f-3ca8d8caa398,DISK] 2024-11-26T14:31:27,604 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1732631487529/wal.1732631487586 2024-11-26T14:31:27,604 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:34431:34431),(127.0.0.1/127.0.0.1:40449:40449),(127.0.0.1/127.0.0.1:35081:35081)] 2024-11-26T14:31:27,604 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 5b9f51ef77304bf0438669929febadea, NAME => 'testReplayEditsWrittenViaHRegion,,1732631487530.5b9f51ef77304bf0438669929febadea.', STARTKEY => '', ENDKEY => ''} 2024-11-26T14:31:27,605 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenViaHRegion,,1732631487530.5b9f51ef77304bf0438669929febadea.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-26T14:31:27,605 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 5b9f51ef77304bf0438669929febadea 2024-11-26T14:31:27,605 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 5b9f51ef77304bf0438669929febadea 2024-11-26T14:31:27,606 INFO [StoreOpener-5b9f51ef77304bf0438669929febadea-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 5b9f51ef77304bf0438669929febadea 2024-11-26T14:31:27,607 INFO [StoreOpener-5b9f51ef77304bf0438669929febadea-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 5b9f51ef77304bf0438669929febadea columnFamilyName a 2024-11-26T14:31:27,607 DEBUG [StoreOpener-5b9f51ef77304bf0438669929febadea-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:31:27,608 INFO [StoreOpener-5b9f51ef77304bf0438669929febadea-1 {}] regionserver.HStore(327): Store=5b9f51ef77304bf0438669929febadea/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-26T14:31:27,608 INFO [StoreOpener-5b9f51ef77304bf0438669929febadea-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 5b9f51ef77304bf0438669929febadea 2024-11-26T14:31:27,609 INFO [StoreOpener-5b9f51ef77304bf0438669929febadea-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 5b9f51ef77304bf0438669929febadea columnFamilyName b 2024-11-26T14:31:27,609 DEBUG [StoreOpener-5b9f51ef77304bf0438669929febadea-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:31:27,610 INFO [StoreOpener-5b9f51ef77304bf0438669929febadea-1 {}] regionserver.HStore(327): Store=5b9f51ef77304bf0438669929febadea/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-26T14:31:27,610 INFO [StoreOpener-5b9f51ef77304bf0438669929febadea-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 5b9f51ef77304bf0438669929febadea 2024-11-26T14:31:27,611 INFO [StoreOpener-5b9f51ef77304bf0438669929febadea-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 5b9f51ef77304bf0438669929febadea columnFamilyName c 2024-11-26T14:31:27,611 DEBUG [StoreOpener-5b9f51ef77304bf0438669929febadea-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:31:27,611 INFO [StoreOpener-5b9f51ef77304bf0438669929febadea-1 {}] regionserver.HStore(327): Store=5b9f51ef77304bf0438669929febadea/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-26T14:31:27,612 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 5b9f51ef77304bf0438669929febadea 2024-11-26T14:31:27,612 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44999/hbase/data/default/testReplayEditsWrittenViaHRegion/5b9f51ef77304bf0438669929febadea 2024-11-26T14:31:27,614 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44999/hbase/data/default/testReplayEditsWrittenViaHRegion/5b9f51ef77304bf0438669929febadea 2024-11-26T14:31:27,615 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 5b9f51ef77304bf0438669929febadea 2024-11-26T14:31:27,615 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 5b9f51ef77304bf0438669929febadea 2024-11-26T14:31:27,615 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsWrittenViaHRegion descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-26T14:31:27,616 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 5b9f51ef77304bf0438669929febadea 2024-11-26T14:31:27,617 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 5b9f51ef77304bf0438669929febadea; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=73747574, jitterRate=0.09892448782920837}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-26T14:31:27,618 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 5b9f51ef77304bf0438669929febadea: Writing region info on filesystem at 1732631487605Initializing all the Stores at 1732631487606 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732631487606Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732631487606Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732631487606Cleaning up temporary data from old regions at 1732631487615 (+9 ms)Region opened successfully at 1732631487618 (+3 ms) 2024-11-26T14:31:27,649 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 5b9f51ef77304bf0438669929febadea 3/3 column families, dataSize=2.55 KB heapSize=5.44 KB 2024-11-26T14:31:27,669 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44999/hbase/data/default/testReplayEditsWrittenViaHRegion/5b9f51ef77304bf0438669929febadea/.tmp/a/ddd0205a552a47b4a655f9e62f08aa4e is 91, key is testReplayEditsWrittenViaHRegion/a:x0/1732631487618/Put/seqid=0 2024-11-26T14:31:27,677 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36277 is added to blk_1073741895_1073 (size=5958) 2024-11-26T14:31:27,677 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38147 is added to blk_1073741895_1073 (size=5958) 2024-11-26T14:31:27,677 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36307 is added to blk_1073741895_1073 (size=5958) 2024-11-26T14:31:27,678 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=33 (bloomFilter=true), to=hdfs://localhost:44999/hbase/data/default/testReplayEditsWrittenViaHRegion/5b9f51ef77304bf0438669929febadea/.tmp/a/ddd0205a552a47b4a655f9e62f08aa4e 2024-11-26T14:31:27,704 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44999/hbase/data/default/testReplayEditsWrittenViaHRegion/5b9f51ef77304bf0438669929febadea/.tmp/b/d9aa4ccc5e1d4bcb9e5e20825f784e68 is 91, key is testReplayEditsWrittenViaHRegion/b:x0/1732631487628/Put/seqid=0 2024-11-26T14:31:27,711 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36277 is added to blk_1073741896_1074 (size=5958) 2024-11-26T14:31:27,712 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36307 is added to blk_1073741896_1074 (size=5958) 2024-11-26T14:31:27,712 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38147 is added to blk_1073741896_1074 (size=5958) 2024-11-26T14:31:27,712 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=33 (bloomFilter=true), to=hdfs://localhost:44999/hbase/data/default/testReplayEditsWrittenViaHRegion/5b9f51ef77304bf0438669929febadea/.tmp/b/d9aa4ccc5e1d4bcb9e5e20825f784e68 2024-11-26T14:31:27,732 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44999/hbase/data/default/testReplayEditsWrittenViaHRegion/5b9f51ef77304bf0438669929febadea/.tmp/c/a804f5afc7a74db9852036e83ca04d0f is 91, key is testReplayEditsWrittenViaHRegion/c:x0/1732631487638/Put/seqid=0 2024-11-26T14:31:27,739 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38147 is added to blk_1073741897_1075 (size=5958) 2024-11-26T14:31:27,739 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36307 is added to blk_1073741897_1075 (size=5958) 2024-11-26T14:31:27,739 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36277 is added to blk_1073741897_1075 (size=5958) 2024-11-26T14:31:27,741 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=33 (bloomFilter=true), to=hdfs://localhost:44999/hbase/data/default/testReplayEditsWrittenViaHRegion/5b9f51ef77304bf0438669929febadea/.tmp/c/a804f5afc7a74db9852036e83ca04d0f 2024-11-26T14:31:27,747 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44999/hbase/data/default/testReplayEditsWrittenViaHRegion/5b9f51ef77304bf0438669929febadea/.tmp/a/ddd0205a552a47b4a655f9e62f08aa4e as hdfs://localhost:44999/hbase/data/default/testReplayEditsWrittenViaHRegion/5b9f51ef77304bf0438669929febadea/a/ddd0205a552a47b4a655f9e62f08aa4e 2024-11-26T14:31:27,754 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44999/hbase/data/default/testReplayEditsWrittenViaHRegion/5b9f51ef77304bf0438669929febadea/a/ddd0205a552a47b4a655f9e62f08aa4e, entries=10, sequenceid=33, filesize=5.8 K 2024-11-26T14:31:27,755 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44999/hbase/data/default/testReplayEditsWrittenViaHRegion/5b9f51ef77304bf0438669929febadea/.tmp/b/d9aa4ccc5e1d4bcb9e5e20825f784e68 as hdfs://localhost:44999/hbase/data/default/testReplayEditsWrittenViaHRegion/5b9f51ef77304bf0438669929febadea/b/d9aa4ccc5e1d4bcb9e5e20825f784e68 2024-11-26T14:31:27,762 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44999/hbase/data/default/testReplayEditsWrittenViaHRegion/5b9f51ef77304bf0438669929febadea/b/d9aa4ccc5e1d4bcb9e5e20825f784e68, entries=10, sequenceid=33, filesize=5.8 K 2024-11-26T14:31:27,763 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44999/hbase/data/default/testReplayEditsWrittenViaHRegion/5b9f51ef77304bf0438669929febadea/.tmp/c/a804f5afc7a74db9852036e83ca04d0f as hdfs://localhost:44999/hbase/data/default/testReplayEditsWrittenViaHRegion/5b9f51ef77304bf0438669929febadea/c/a804f5afc7a74db9852036e83ca04d0f 2024-11-26T14:31:27,771 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44999/hbase/data/default/testReplayEditsWrittenViaHRegion/5b9f51ef77304bf0438669929febadea/c/a804f5afc7a74db9852036e83ca04d0f, entries=10, sequenceid=33, filesize=5.8 K 2024-11-26T14:31:27,773 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~2.55 KB/2610, heapSize ~5.39 KB/5520, currentSize=0 B/0 for 5b9f51ef77304bf0438669929febadea in 123ms, sequenceid=33, compaction requested=false 2024-11-26T14:31:27,773 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for 5b9f51ef77304bf0438669929febadea: 2024-11-26T14:31:27,773 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 5b9f51ef77304bf0438669929febadea, disabling compactions & flushes 2024-11-26T14:31:27,773 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsWrittenViaHRegion,,1732631487530.5b9f51ef77304bf0438669929febadea. 2024-11-26T14:31:27,773 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsWrittenViaHRegion,,1732631487530.5b9f51ef77304bf0438669929febadea. 2024-11-26T14:31:27,773 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsWrittenViaHRegion,,1732631487530.5b9f51ef77304bf0438669929febadea. after waiting 0 ms 2024-11-26T14:31:27,773 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsWrittenViaHRegion,,1732631487530.5b9f51ef77304bf0438669929febadea. 2024-11-26T14:31:27,774 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsWrittenViaHRegion,,1732631487530.5b9f51ef77304bf0438669929febadea. 2024-11-26T14:31:27,775 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 5b9f51ef77304bf0438669929febadea: Waiting for close lock at 1732631487773Disabling compacts and flushes for region at 1732631487773Disabling writes for close at 1732631487773Writing region close event to WAL at 1732631487774 (+1 ms)Closed at 1732631487774 2024-11-26T14:31:27,777 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38147 is added to blk_1073741894_1072 (size=2873) 2024-11-26T14:31:27,777 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36307 is added to blk_1073741894_1072 (size=2873) 2024-11-26T14:31:27,778 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36277 is added to blk_1073741894_1072 (size=2873) 2024-11-26T14:31:27,782 DEBUG [Time-limited test {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44999/hbase/data/default/testReplayEditsWrittenViaHRegion/5b9f51ef77304bf0438669929febadea/b/d9aa4ccc5e1d4bcb9e5e20825f784e68 to hdfs://localhost:44999/hbase/archive/data/default/testReplayEditsWrittenViaHRegion/5b9f51ef77304bf0438669929febadea/b/d9aa4ccc5e1d4bcb9e5e20825f784e68 2024-11-26T14:31:27,805 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:44999/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1732631487529/wal.1732631487586, size=2.8 K (2873bytes) 2024-11-26T14:31:27,805 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:44999/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1732631487529/wal.1732631487586 2024-11-26T14:31:27,805 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:44999/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1732631487529/wal.1732631487586 after 0ms 2024-11-26T14:31:27,808 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:44999/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1732631487529/wal.1732631487586: isRecoveredEdits=false, hasTagCompression=true, hasValueCompression=false, valueCompressionType=null 2024-11-26T14:31:27,808 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:44999/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1732631487529/wal.1732631487586 took 4ms 2024-11-26T14:31:27,811 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:44999/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1732631487529/wal.1732631487586 so closing down 2024-11-26T14:31:27,811 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-26T14:31:27,812 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000003-wal.1732631487586.temp 2024-11-26T14:31:27,813 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:44999/hbase/data/default/testReplayEditsWrittenViaHRegion/5b9f51ef77304bf0438669929febadea/recovered.edits/0000000000000000003-wal.1732631487586.temp 2024-11-26T14:31:27,814 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-11-26T14:31:27,820 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38147 is added to blk_1073741898_1076 (size=2312) 2024-11-26T14:31:27,820 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36307 is added to blk_1073741898_1076 (size=2312) 2024-11-26T14:31:27,821 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36277 is added to blk_1073741898_1076 (size=2312) 2024-11-26T14:31:27,822 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:44999/hbase/data/default/testReplayEditsWrittenViaHRegion/5b9f51ef77304bf0438669929febadea/recovered.edits/0000000000000000003-wal.1732631487586.temp (wrote 30 edits, skipped 0 edits in 0 ms) 2024-11-26T14:31:27,824 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:44999/hbase/data/default/testReplayEditsWrittenViaHRegion/5b9f51ef77304bf0438669929febadea/recovered.edits/0000000000000000003-wal.1732631487586.temp to hdfs://localhost:44999/hbase/data/default/testReplayEditsWrittenViaHRegion/5b9f51ef77304bf0438669929febadea/recovered.edits/0000000000000000032 2024-11-26T14:31:27,824 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 32 edits across 1 Regions in 15 ms; skipped=2; WAL=hdfs://localhost:44999/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1732631487529/wal.1732631487586, size=2.8 K, length=2873, corrupted=false, cancelled=false 2024-11-26T14:31:27,824 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:44999/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1732631487529/wal.1732631487586, journal: Splitting hdfs://localhost:44999/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1732631487529/wal.1732631487586, size=2.8 K (2873bytes) at 1732631487805Finishing writing output for hdfs://localhost:44999/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1732631487529/wal.1732631487586 so closing down at 1732631487811 (+6 ms)Creating recovered edits writer path=hdfs://localhost:44999/hbase/data/default/testReplayEditsWrittenViaHRegion/5b9f51ef77304bf0438669929febadea/recovered.edits/0000000000000000003-wal.1732631487586.temp at 1732631487813 (+2 ms)3 split writer threads finished at 1732631487814 (+1 ms)Closed recovered edits writer path=hdfs://localhost:44999/hbase/data/default/testReplayEditsWrittenViaHRegion/5b9f51ef77304bf0438669929febadea/recovered.edits/0000000000000000003-wal.1732631487586.temp (wrote 30 edits, skipped 0 edits in 0 ms) at 1732631487822 (+8 ms)Rename recovered edits hdfs://localhost:44999/hbase/data/default/testReplayEditsWrittenViaHRegion/5b9f51ef77304bf0438669929febadea/recovered.edits/0000000000000000003-wal.1732631487586.temp to hdfs://localhost:44999/hbase/data/default/testReplayEditsWrittenViaHRegion/5b9f51ef77304bf0438669929febadea/recovered.edits/0000000000000000032 at 1732631487824 (+2 ms)Processed 32 edits across 1 Regions in 15 ms; skipped=2; WAL=hdfs://localhost:44999/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1732631487529/wal.1732631487586, size=2.8 K, length=2873, corrupted=false, cancelled=false at 1732631487824 2024-11-26T14:31:27,826 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:44999/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1732631487529/wal.1732631487586 to hdfs://localhost:44999/hbase/oldWALs/wal.1732631487586 2024-11-26T14:31:27,827 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:44999/hbase/data/default/testReplayEditsWrittenViaHRegion/5b9f51ef77304bf0438669929febadea/recovered.edits/0000000000000000032 2024-11-26T14:31:27,827 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-26T14:31:27,829 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:44999/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1732631487529, archiveDir=hdfs://localhost:44999/hbase/oldWALs, maxLogs=32 2024-11-26T14:31:27,847 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1732631487529/wal.1732631487829, exclude list is [], retry=0 2024-11-26T14:31:27,850 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36277,DS-9cb9383d-a6c8-4ba6-83b3-6706331a6539,DISK] 2024-11-26T14:31:27,851 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38147,DS-44b12d44-38d0-4c76-b56f-3ca8d8caa398,DISK] 2024-11-26T14:31:27,851 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36307,DS-dd37de78-b3d5-4d7c-8223-4b15101c21bf,DISK] 2024-11-26T14:31:27,945 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1732631487529/wal.1732631487829 2024-11-26T14:31:27,946 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:40449:40449),(127.0.0.1/127.0.0.1:35081:35081),(127.0.0.1/127.0.0.1:34431:34431)] 2024-11-26T14:31:27,946 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 5b9f51ef77304bf0438669929febadea, NAME => 'testReplayEditsWrittenViaHRegion,,1732631487530.5b9f51ef77304bf0438669929febadea.', STARTKEY => '', ENDKEY => ''} 2024-11-26T14:31:27,946 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenViaHRegion,,1732631487530.5b9f51ef77304bf0438669929febadea.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-26T14:31:27,946 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 5b9f51ef77304bf0438669929febadea 2024-11-26T14:31:27,946 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 5b9f51ef77304bf0438669929febadea 2024-11-26T14:31:27,948 INFO [StoreOpener-5b9f51ef77304bf0438669929febadea-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 5b9f51ef77304bf0438669929febadea 2024-11-26T14:31:27,950 INFO [StoreOpener-5b9f51ef77304bf0438669929febadea-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 5b9f51ef77304bf0438669929febadea columnFamilyName a 2024-11-26T14:31:27,950 DEBUG [StoreOpener-5b9f51ef77304bf0438669929febadea-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:31:27,958 DEBUG [StoreOpener-5b9f51ef77304bf0438669929febadea-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:44999/hbase/data/default/testReplayEditsWrittenViaHRegion/5b9f51ef77304bf0438669929febadea/a/ddd0205a552a47b4a655f9e62f08aa4e 2024-11-26T14:31:27,958 INFO [StoreOpener-5b9f51ef77304bf0438669929febadea-1 {}] regionserver.HStore(327): Store=5b9f51ef77304bf0438669929febadea/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-26T14:31:27,958 INFO [StoreOpener-5b9f51ef77304bf0438669929febadea-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 5b9f51ef77304bf0438669929febadea 2024-11-26T14:31:27,959 INFO [StoreOpener-5b9f51ef77304bf0438669929febadea-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 5b9f51ef77304bf0438669929febadea columnFamilyName b 2024-11-26T14:31:27,960 DEBUG [StoreOpener-5b9f51ef77304bf0438669929febadea-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:31:27,960 INFO [StoreOpener-5b9f51ef77304bf0438669929febadea-1 {}] regionserver.HStore(327): Store=5b9f51ef77304bf0438669929febadea/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-26T14:31:27,960 INFO [StoreOpener-5b9f51ef77304bf0438669929febadea-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 5b9f51ef77304bf0438669929febadea 2024-11-26T14:31:27,961 INFO [StoreOpener-5b9f51ef77304bf0438669929febadea-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 5b9f51ef77304bf0438669929febadea columnFamilyName c 2024-11-26T14:31:27,961 DEBUG [StoreOpener-5b9f51ef77304bf0438669929febadea-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:31:27,967 DEBUG [StoreOpener-5b9f51ef77304bf0438669929febadea-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:44999/hbase/data/default/testReplayEditsWrittenViaHRegion/5b9f51ef77304bf0438669929febadea/c/a804f5afc7a74db9852036e83ca04d0f 2024-11-26T14:31:27,967 INFO [StoreOpener-5b9f51ef77304bf0438669929febadea-1 {}] regionserver.HStore(327): Store=5b9f51ef77304bf0438669929febadea/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-26T14:31:27,967 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 5b9f51ef77304bf0438669929febadea 2024-11-26T14:31:27,968 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44999/hbase/data/default/testReplayEditsWrittenViaHRegion/5b9f51ef77304bf0438669929febadea 2024-11-26T14:31:27,970 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:44999/hbase/data/default/testReplayEditsWrittenViaHRegion/5b9f51ef77304bf0438669929febadea 2024-11-26T14:31:27,971 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:44999/hbase/data/default/testReplayEditsWrittenViaHRegion/5b9f51ef77304bf0438669929febadea/recovered.edits/0000000000000000032 2024-11-26T14:31:27,973 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:44999/hbase/data/default/testReplayEditsWrittenViaHRegion/5b9f51ef77304bf0438669929febadea/recovered.edits/0000000000000000032: isRecoveredEdits=true, hasTagCompression=true, hasValueCompression=false, valueCompressionType=null 2024-11-26T14:31:27,974 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 10, skipped 20, firstSequenceIdInLog=3, maxSequenceIdInLog=32, path=hdfs://localhost:44999/hbase/data/default/testReplayEditsWrittenViaHRegion/5b9f51ef77304bf0438669929febadea/recovered.edits/0000000000000000032 2024-11-26T14:31:27,974 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 5b9f51ef77304bf0438669929febadea 3/3 column families, dataSize=870 B heapSize=2.31 KB 2024-11-26T14:31:27,989 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44999/hbase/data/default/testReplayEditsWrittenViaHRegion/5b9f51ef77304bf0438669929febadea/.tmp/b/ded47b2021d94012a03c58eca0b82ade is 91, key is testReplayEditsWrittenViaHRegion/b:x0/1732631487628/Put/seqid=0 2024-11-26T14:31:27,995 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36277 is added to blk_1073741900_1078 (size=5958) 2024-11-26T14:31:27,995 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36307 is added to blk_1073741900_1078 (size=5958) 2024-11-26T14:31:27,995 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38147 is added to blk_1073741900_1078 (size=5958) 2024-11-26T14:31:27,996 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=32 (bloomFilter=true), to=hdfs://localhost:44999/hbase/data/default/testReplayEditsWrittenViaHRegion/5b9f51ef77304bf0438669929febadea/.tmp/b/ded47b2021d94012a03c58eca0b82ade 2024-11-26T14:31:28,001 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44999/hbase/data/default/testReplayEditsWrittenViaHRegion/5b9f51ef77304bf0438669929febadea/.tmp/b/ded47b2021d94012a03c58eca0b82ade as hdfs://localhost:44999/hbase/data/default/testReplayEditsWrittenViaHRegion/5b9f51ef77304bf0438669929febadea/b/ded47b2021d94012a03c58eca0b82ade 2024-11-26T14:31:28,006 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44999/hbase/data/default/testReplayEditsWrittenViaHRegion/5b9f51ef77304bf0438669929febadea/b/ded47b2021d94012a03c58eca0b82ade, entries=10, sequenceid=32, filesize=5.8 K 2024-11-26T14:31:28,007 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~870 B/870, heapSize ~1.80 KB/1840, currentSize=0 B/0 for 5b9f51ef77304bf0438669929febadea in 33ms, sequenceid=32, compaction requested=false; wal=null 2024-11-26T14:31:28,007 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:44999/hbase/data/default/testReplayEditsWrittenViaHRegion/5b9f51ef77304bf0438669929febadea/recovered.edits/0000000000000000032 2024-11-26T14:31:28,008 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 5b9f51ef77304bf0438669929febadea 2024-11-26T14:31:28,008 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 5b9f51ef77304bf0438669929febadea 2024-11-26T14:31:28,009 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsWrittenViaHRegion descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-26T14:31:28,011 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 5b9f51ef77304bf0438669929febadea 2024-11-26T14:31:28,013 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44999/hbase/data/default/testReplayEditsWrittenViaHRegion/5b9f51ef77304bf0438669929febadea/recovered.edits/33.seqid, newMaxSeqId=33, maxSeqId=1 2024-11-26T14:31:28,014 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 5b9f51ef77304bf0438669929febadea; next sequenceid=34; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=66663564, jitterRate=-0.006635487079620361}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-26T14:31:28,015 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 5b9f51ef77304bf0438669929febadea: Writing region info on filesystem at 1732631487947Initializing all the Stores at 1732631487948 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732631487948Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732631487948Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732631487948Obtaining lock to block concurrent updates at 1732631487975 (+27 ms)Preparing flush snapshotting stores in 5b9f51ef77304bf0438669929febadea at 1732631487975Finished memstore snapshotting testReplayEditsWrittenViaHRegion,,1732631487530.5b9f51ef77304bf0438669929febadea., syncing WAL and waiting on mvcc, flushsize=dataSize=870, getHeapSize=2320, getOffHeapSize=0, getCellsCount=10 at 1732631487975Flushing stores of testReplayEditsWrittenViaHRegion,,1732631487530.5b9f51ef77304bf0438669929febadea. at 1732631487975Flushing 5b9f51ef77304bf0438669929febadea/b: creating writer at 1732631487975Flushing 5b9f51ef77304bf0438669929febadea/b: appending metadata at 1732631487988 (+13 ms)Flushing 5b9f51ef77304bf0438669929febadea/b: closing flushed file at 1732631487989 (+1 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@17d83cf5: reopening flushed file at 1732631488000 (+11 ms)Finished flush of dataSize ~870 B/870, heapSize ~1.80 KB/1840, currentSize=0 B/0 for 5b9f51ef77304bf0438669929febadea in 33ms, sequenceid=32, compaction requested=false; wal=null at 1732631488007 (+7 ms)Cleaning up temporary data from old regions at 1732631488008 (+1 ms)Region opened successfully at 1732631488015 (+7 ms) 2024-11-26T14:31:28,039 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestAsyncWALReplayCompressed#testReplayEditsAfterPartialFlush Thread=424 (was 415) Potentially hanging thread: PacketResponder: BP-1347678837-172.17.0.3-1732631464290:blk_1073741899_1077, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_2043263076_22 at /127.0.0.1:39498 [Waiting for operation #36] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_2043263076_22 at /127.0.0.1:39130 [Waiting for operation #31] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-1347678837-172.17.0.3-1732631464290:blk_1073741899_1077, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-1347678837-172.17.0.3-1732631464290:blk_1073741899_1077, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-20-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_2043263076_22 at /127.0.0.1:39274 [Receiving block BP-1347678837-172.17.0.3-1732631464290:blk_1073741899_1077] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_2043263076_22 at /127.0.0.1:39666 [Receiving block BP-1347678837-172.17.0.3-1732631464290:blk_1073741899_1077] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-20-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_2043263076_22 at /127.0.0.1:43760 [Waiting for operation #35] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-20-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_2043263076_22 at /127.0.0.1:43910 [Receiving block BP-1347678837-172.17.0.3-1732631464290:blk_1073741899_1077] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=1127 (was 1051) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=260 (was 248) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=9029 (was 9058) 2024-11-26T14:31:28,039 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1127 is superior to 1024 2024-11-26T14:31:28,054 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestAsyncWALReplayCompressed#testReplayEditsAfterAbortingFlush Thread=424, OpenFileDescriptor=1127, MaxFileDescriptor=1048576, SystemLoadAverage=260, ProcessCount=11, AvailableMemoryMB=9029 2024-11-26T14:31:28,054 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1127 is superior to 1024 2024-11-26T14:31:28,071 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-26T14:31:28,074 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-26T14:31:28,074 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-26T14:31:28,077 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-39086878, suffix=, logDir=hdfs://localhost:44999/hbase/WALs/hregion-39086878, archiveDir=hdfs://localhost:44999/hbase/oldWALs, maxLogs=32 2024-11-26T14:31:28,090 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-39086878/hregion-39086878.1732631488077, exclude list is [], retry=0 2024-11-26T14:31:28,093 DEBUG [AsyncFSWAL-22-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38147,DS-44b12d44-38d0-4c76-b56f-3ca8d8caa398,DISK] 2024-11-26T14:31:28,093 DEBUG [AsyncFSWAL-22-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36277,DS-9cb9383d-a6c8-4ba6-83b3-6706331a6539,DISK] 2024-11-26T14:31:28,094 DEBUG [AsyncFSWAL-22-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36307,DS-dd37de78-b3d5-4d7c-8223-4b15101c21bf,DISK] 2024-11-26T14:31:28,097 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-39086878/hregion-39086878.1732631488077 2024-11-26T14:31:28,097 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:35081:35081),(127.0.0.1/127.0.0.1:40449:40449),(127.0.0.1/127.0.0.1:34431:34431)] 2024-11-26T14:31:28,098 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => 6c4b2f0afc1fab59a8a3937aad627963, NAME => 'testReplayEditsAfterAbortingFlush,,1732631488072.6c4b2f0afc1fab59a8a3937aad627963.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testReplayEditsAfterAbortingFlush', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:44999/hbase 2024-11-26T14:31:28,110 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38147 is added to blk_1073741902_1080 (size=68) 2024-11-26T14:31:28,110 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36307 is added to blk_1073741902_1080 (size=68) 2024-11-26T14:31:28,110 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36277 is added to blk_1073741902_1080 (size=68) 2024-11-26T14:31:28,111 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsAfterAbortingFlush,,1732631488072.6c4b2f0afc1fab59a8a3937aad627963.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-26T14:31:28,112 INFO [StoreOpener-6c4b2f0afc1fab59a8a3937aad627963-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 6c4b2f0afc1fab59a8a3937aad627963 2024-11-26T14:31:28,114 INFO [StoreOpener-6c4b2f0afc1fab59a8a3937aad627963-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 6c4b2f0afc1fab59a8a3937aad627963 columnFamilyName a 2024-11-26T14:31:28,114 DEBUG [StoreOpener-6c4b2f0afc1fab59a8a3937aad627963-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:31:28,114 INFO [StoreOpener-6c4b2f0afc1fab59a8a3937aad627963-1 {}] regionserver.HStore(327): Store=6c4b2f0afc1fab59a8a3937aad627963/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-26T14:31:28,115 INFO [StoreOpener-6c4b2f0afc1fab59a8a3937aad627963-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 6c4b2f0afc1fab59a8a3937aad627963 2024-11-26T14:31:28,116 INFO [StoreOpener-6c4b2f0afc1fab59a8a3937aad627963-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 6c4b2f0afc1fab59a8a3937aad627963 columnFamilyName b 2024-11-26T14:31:28,116 DEBUG [StoreOpener-6c4b2f0afc1fab59a8a3937aad627963-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:31:28,117 INFO [StoreOpener-6c4b2f0afc1fab59a8a3937aad627963-1 {}] regionserver.HStore(327): Store=6c4b2f0afc1fab59a8a3937aad627963/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-26T14:31:28,117 INFO [StoreOpener-6c4b2f0afc1fab59a8a3937aad627963-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 6c4b2f0afc1fab59a8a3937aad627963 2024-11-26T14:31:28,118 INFO [StoreOpener-6c4b2f0afc1fab59a8a3937aad627963-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 6c4b2f0afc1fab59a8a3937aad627963 columnFamilyName c 2024-11-26T14:31:28,119 DEBUG [StoreOpener-6c4b2f0afc1fab59a8a3937aad627963-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:31:28,119 INFO [StoreOpener-6c4b2f0afc1fab59a8a3937aad627963-1 {}] regionserver.HStore(327): Store=6c4b2f0afc1fab59a8a3937aad627963/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-26T14:31:28,119 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 6c4b2f0afc1fab59a8a3937aad627963 2024-11-26T14:31:28,120 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44999/hbase/data/default/testReplayEditsAfterAbortingFlush/6c4b2f0afc1fab59a8a3937aad627963 2024-11-26T14:31:28,121 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44999/hbase/data/default/testReplayEditsAfterAbortingFlush/6c4b2f0afc1fab59a8a3937aad627963 2024-11-26T14:31:28,122 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 6c4b2f0afc1fab59a8a3937aad627963 2024-11-26T14:31:28,122 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 6c4b2f0afc1fab59a8a3937aad627963 2024-11-26T14:31:28,122 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsAfterAbortingFlush descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-26T14:31:28,124 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 6c4b2f0afc1fab59a8a3937aad627963 2024-11-26T14:31:28,126 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44999/hbase/data/default/testReplayEditsAfterAbortingFlush/6c4b2f0afc1fab59a8a3937aad627963/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-26T14:31:28,127 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 6c4b2f0afc1fab59a8a3937aad627963; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=69431414, jitterRate=0.034608691930770874}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-26T14:31:28,128 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 6c4b2f0afc1fab59a8a3937aad627963: Writing region info on filesystem at 1732631488111Initializing all the Stores at 1732631488112 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732631488112Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732631488112Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732631488112Cleaning up temporary data from old regions at 1732631488122 (+10 ms)Region opened successfully at 1732631488127 (+5 ms) 2024-11-26T14:31:28,128 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 6c4b2f0afc1fab59a8a3937aad627963, disabling compactions & flushes 2024-11-26T14:31:28,128 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsAfterAbortingFlush,,1732631488072.6c4b2f0afc1fab59a8a3937aad627963. 2024-11-26T14:31:28,128 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsAfterAbortingFlush,,1732631488072.6c4b2f0afc1fab59a8a3937aad627963. 2024-11-26T14:31:28,128 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsAfterAbortingFlush,,1732631488072.6c4b2f0afc1fab59a8a3937aad627963. after waiting 0 ms 2024-11-26T14:31:28,128 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsAfterAbortingFlush,,1732631488072.6c4b2f0afc1fab59a8a3937aad627963. 2024-11-26T14:31:28,128 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsAfterAbortingFlush,,1732631488072.6c4b2f0afc1fab59a8a3937aad627963. 2024-11-26T14:31:28,129 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 6c4b2f0afc1fab59a8a3937aad627963: Waiting for close lock at 1732631488128Disabling compacts and flushes for region at 1732631488128Disabling writes for close at 1732631488128Writing region close event to WAL at 1732631488128Closed at 1732631488128 2024-11-26T14:31:28,134 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36307 is added to blk_1073741901_1079 (size=93) 2024-11-26T14:31:28,134 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38147 is added to blk_1073741901_1079 (size=93) 2024-11-26T14:31:28,135 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36277 is added to blk_1073741901_1079 (size=93) 2024-11-26T14:31:28,137 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-11-26T14:31:28,137 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-39086878:(num 1732631488077) 2024-11-26T14:31:28,137 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-26T14:31:28,139 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:44999/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1732631488070, archiveDir=hdfs://localhost:44999/hbase/oldWALs, maxLogs=32 2024-11-26T14:31:28,152 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1732631488070/wal.1732631488140, exclude list is [], retry=0 2024-11-26T14:31:28,154 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36307,DS-dd37de78-b3d5-4d7c-8223-4b15101c21bf,DISK] 2024-11-26T14:31:28,155 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36277,DS-9cb9383d-a6c8-4ba6-83b3-6706331a6539,DISK] 2024-11-26T14:31:28,155 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38147,DS-44b12d44-38d0-4c76-b56f-3ca8d8caa398,DISK] 2024-11-26T14:31:28,157 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1732631488070/wal.1732631488140 2024-11-26T14:31:28,157 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:34431:34431),(127.0.0.1/127.0.0.1:40449:40449),(127.0.0.1/127.0.0.1:35081:35081)] 2024-11-26T14:31:28,216 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 6c4b2f0afc1fab59a8a3937aad627963, NAME => 'testReplayEditsAfterAbortingFlush,,1732631488072.6c4b2f0afc1fab59a8a3937aad627963.', STARTKEY => '', ENDKEY => ''} 2024-11-26T14:31:28,219 DEBUG [Time-limited test {}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testReplayEditsAfterAbortingFlush 6c4b2f0afc1fab59a8a3937aad627963 2024-11-26T14:31:28,219 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsAfterAbortingFlush,,1732631488072.6c4b2f0afc1fab59a8a3937aad627963.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-26T14:31:28,219 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 6c4b2f0afc1fab59a8a3937aad627963 2024-11-26T14:31:28,219 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 6c4b2f0afc1fab59a8a3937aad627963 2024-11-26T14:31:28,221 INFO [StoreOpener-6c4b2f0afc1fab59a8a3937aad627963-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 6c4b2f0afc1fab59a8a3937aad627963 2024-11-26T14:31:28,222 INFO [StoreOpener-6c4b2f0afc1fab59a8a3937aad627963-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 6c4b2f0afc1fab59a8a3937aad627963 columnFamilyName a 2024-11-26T14:31:28,222 DEBUG [StoreOpener-6c4b2f0afc1fab59a8a3937aad627963-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:31:28,223 INFO [StoreOpener-6c4b2f0afc1fab59a8a3937aad627963-1 {}] regionserver.HStore(327): Store=6c4b2f0afc1fab59a8a3937aad627963/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-26T14:31:28,223 INFO [StoreOpener-6c4b2f0afc1fab59a8a3937aad627963-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 6c4b2f0afc1fab59a8a3937aad627963 2024-11-26T14:31:28,224 INFO [StoreOpener-6c4b2f0afc1fab59a8a3937aad627963-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 6c4b2f0afc1fab59a8a3937aad627963 columnFamilyName b 2024-11-26T14:31:28,224 DEBUG [StoreOpener-6c4b2f0afc1fab59a8a3937aad627963-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:31:28,225 INFO [StoreOpener-6c4b2f0afc1fab59a8a3937aad627963-1 {}] regionserver.HStore(327): Store=6c4b2f0afc1fab59a8a3937aad627963/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-26T14:31:28,225 INFO [StoreOpener-6c4b2f0afc1fab59a8a3937aad627963-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 6c4b2f0afc1fab59a8a3937aad627963 2024-11-26T14:31:28,226 INFO [StoreOpener-6c4b2f0afc1fab59a8a3937aad627963-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 6c4b2f0afc1fab59a8a3937aad627963 columnFamilyName c 2024-11-26T14:31:28,226 DEBUG [StoreOpener-6c4b2f0afc1fab59a8a3937aad627963-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:31:28,227 INFO [StoreOpener-6c4b2f0afc1fab59a8a3937aad627963-1 {}] regionserver.HStore(327): Store=6c4b2f0afc1fab59a8a3937aad627963/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-26T14:31:28,227 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 6c4b2f0afc1fab59a8a3937aad627963 2024-11-26T14:31:28,227 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44999/hbase/data/default/testReplayEditsAfterAbortingFlush/6c4b2f0afc1fab59a8a3937aad627963 2024-11-26T14:31:28,229 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44999/hbase/data/default/testReplayEditsAfterAbortingFlush/6c4b2f0afc1fab59a8a3937aad627963 2024-11-26T14:31:28,229 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 6c4b2f0afc1fab59a8a3937aad627963 2024-11-26T14:31:28,230 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 6c4b2f0afc1fab59a8a3937aad627963 2024-11-26T14:31:28,230 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsAfterAbortingFlush descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-26T14:31:28,231 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 6c4b2f0afc1fab59a8a3937aad627963 2024-11-26T14:31:28,232 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 6c4b2f0afc1fab59a8a3937aad627963; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=68074015, jitterRate=0.01438187062740326}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-26T14:31:28,232 DEBUG [Time-limited test {}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 6c4b2f0afc1fab59a8a3937aad627963 2024-11-26T14:31:28,233 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 6c4b2f0afc1fab59a8a3937aad627963: Running coprocessor pre-open hook at 1732631488219Writing region info on filesystem at 1732631488219Initializing all the Stores at 1732631488220 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732631488220Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732631488221 (+1 ms)Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732631488221Cleaning up temporary data from old regions at 1732631488230 (+9 ms)Running coprocessor post-open hooks at 1732631488232 (+2 ms)Region opened successfully at 1732631488233 (+1 ms) 2024-11-26T14:31:28,246 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 6c4b2f0afc1fab59a8a3937aad627963 3/3 column families, dataSize=590 B heapSize=2.08 KB 2024-11-26T14:31:28,247 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for 6c4b2f0afc1fab59a8a3937aad627963/a, retrying num=0 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T14:31:28,994 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testReplayEditsWrittenViaHRegion 2024-11-26T14:31:28,995 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testReplayEditsWrittenViaHRegion Metrics about Tables on a single HBase RegionServer 2024-11-26T14:31:28,998 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testSequentialEditLogSeqNum 2024-11-26T14:31:28,998 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testSequentialEditLogSeqNum Metrics about Tables on a single HBase RegionServer 2024-11-26T14:31:29,000 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testReplayEditsAfterRegionMovedWithMultiCF 2024-11-26T14:31:29,000 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testReplayEditsAfterRegionMovedWithMultiCF Metrics about Tables on a single HBase RegionServer 2024-11-26T14:31:29,001 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testRegionMadeOfBulkLoadedFilesOnly 2024-11-26T14:31:29,001 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testRegionMadeOfBulkLoadedFilesOnly Metrics about Tables on a single HBase RegionServer 2024-11-26T14:31:29,003 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testReplayEditsAfterAbortingFlush 2024-11-26T14:31:29,003 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testReplayEditsAfterAbortingFlush Metrics about Tables on a single HBase RegionServer 2024-11-26T14:31:29,248 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for 6c4b2f0afc1fab59a8a3937aad627963/a, retrying num=1 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T14:31:30,029 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-26T14:31:30,249 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for 6c4b2f0afc1fab59a8a3937aad627963/a, retrying num=2 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T14:31:31,251 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for 6c4b2f0afc1fab59a8a3937aad627963/a, retrying num=3 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T14:31:32,252 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for 6c4b2f0afc1fab59a8a3937aad627963/a, retrying num=4 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T14:31:33,253 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for 6c4b2f0afc1fab59a8a3937aad627963/a, retrying num=5 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T14:31:34,255 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for 6c4b2f0afc1fab59a8a3937aad627963/a, retrying num=6 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T14:31:35,256 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for 6c4b2f0afc1fab59a8a3937aad627963/a, retrying num=7 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T14:31:36,257 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for 6c4b2f0afc1fab59a8a3937aad627963/a, retrying num=8 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T14:31:37,258 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for 6c4b2f0afc1fab59a8a3937aad627963/a, retrying num=9 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T14:31:37,261 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for 6c4b2f0afc1fab59a8a3937aad627963: 2024-11-26T14:31:37,261 INFO [Time-limited test {}] wal.AbstractTestWALReplay(671): Expected simulated exception when flushing region, region: testReplayEditsAfterAbortingFlush,,1732631488072.6c4b2f0afc1fab59a8a3937aad627963. 2024-11-26T14:31:37,281 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for 6c4b2f0afc1fab59a8a3937aad627963: 2024-11-26T14:31:37,281 INFO [Time-limited test {}] wal.AbstractTestWALReplay(691): Expected exception when flushing region because server is stopped,Aborting flush because server is aborted... 2024-11-26T14:31:37,281 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 6c4b2f0afc1fab59a8a3937aad627963, disabling compactions & flushes 2024-11-26T14:31:37,281 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsAfterAbortingFlush,,1732631488072.6c4b2f0afc1fab59a8a3937aad627963. 2024-11-26T14:31:37,281 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsAfterAbortingFlush,,1732631488072.6c4b2f0afc1fab59a8a3937aad627963. 2024-11-26T14:31:37,281 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsAfterAbortingFlush,,1732631488072.6c4b2f0afc1fab59a8a3937aad627963. after waiting 0 ms 2024-11-26T14:31:37,281 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsAfterAbortingFlush,,1732631488072.6c4b2f0afc1fab59a8a3937aad627963. 2024-11-26T14:31:37,282 ERROR [Time-limited test {}] regionserver.HRegion(1960): Memstore data size is 1190 in region testReplayEditsAfterAbortingFlush,,1732631488072.6c4b2f0afc1fab59a8a3937aad627963. 2024-11-26T14:31:37,282 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsAfterAbortingFlush,,1732631488072.6c4b2f0afc1fab59a8a3937aad627963. 2024-11-26T14:31:37,282 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 6c4b2f0afc1fab59a8a3937aad627963: Waiting for close lock at 1732631497281Running coprocessor pre-close hooks at 1732631497281Disabling compacts and flushes for region at 1732631497281Disabling writes for close at 1732631497281Writing region close event to WAL at 1732631497282 (+1 ms)Running coprocessor post-close hooks at 1732631497282Closed at 1732631497282 2024-11-26T14:31:37,285 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36307 is added to blk_1073741903_1081 (size=2398) 2024-11-26T14:31:37,285 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38147 is added to blk_1073741903_1081 (size=2398) 2024-11-26T14:31:37,285 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36277 is added to blk_1073741903_1081 (size=2398) 2024-11-26T14:31:37,300 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:44999/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1732631488070/wal.1732631488140, size=2.3 K (2398bytes) 2024-11-26T14:31:37,300 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:44999/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1732631488070/wal.1732631488140 2024-11-26T14:31:37,301 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:44999/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1732631488070/wal.1732631488140 after 1ms 2024-11-26T14:31:37,303 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:44999/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1732631488070/wal.1732631488140: isRecoveredEdits=false, hasTagCompression=true, hasValueCompression=false, valueCompressionType=null 2024-11-26T14:31:37,304 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:44999/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1732631488070/wal.1732631488140 took 4ms 2024-11-26T14:31:37,306 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:44999/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1732631488070/wal.1732631488140 so closing down 2024-11-26T14:31:37,306 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-26T14:31:37,307 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000004-wal.1732631488140.temp 2024-11-26T14:31:37,308 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:44999/hbase/data/default/testReplayEditsAfterAbortingFlush/6c4b2f0afc1fab59a8a3937aad627963/recovered.edits/0000000000000000004-wal.1732631488140.temp 2024-11-26T14:31:37,309 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-11-26T14:31:37,318 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38147 is added to blk_1073741904_1082 (size=1672) 2024-11-26T14:31:37,318 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36307 is added to blk_1073741904_1082 (size=1672) 2024-11-26T14:31:37,318 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36277 is added to blk_1073741904_1082 (size=1672) 2024-11-26T14:31:37,319 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:44999/hbase/data/default/testReplayEditsAfterAbortingFlush/6c4b2f0afc1fab59a8a3937aad627963/recovered.edits/0000000000000000004-wal.1732631488140.temp (wrote 20 edits, skipped 0 edits in 0 ms) 2024-11-26T14:31:37,320 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:44999/hbase/data/default/testReplayEditsAfterAbortingFlush/6c4b2f0afc1fab59a8a3937aad627963/recovered.edits/0000000000000000004-wal.1732631488140.temp to hdfs://localhost:44999/hbase/data/default/testReplayEditsAfterAbortingFlush/6c4b2f0afc1fab59a8a3937aad627963/recovered.edits/0000000000000000026 2024-11-26T14:31:37,320 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 23 edits across 1 Regions in 16 ms; skipped=3; WAL=hdfs://localhost:44999/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1732631488070/wal.1732631488140, size=2.3 K, length=2398, corrupted=false, cancelled=false 2024-11-26T14:31:37,320 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:44999/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1732631488070/wal.1732631488140, journal: Splitting hdfs://localhost:44999/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1732631488070/wal.1732631488140, size=2.3 K (2398bytes) at 1732631497300Finishing writing output for hdfs://localhost:44999/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1732631488070/wal.1732631488140 so closing down at 1732631497306 (+6 ms)Creating recovered edits writer path=hdfs://localhost:44999/hbase/data/default/testReplayEditsAfterAbortingFlush/6c4b2f0afc1fab59a8a3937aad627963/recovered.edits/0000000000000000004-wal.1732631488140.temp at 1732631497308 (+2 ms)3 split writer threads finished at 1732631497309 (+1 ms)Closed recovered edits writer path=hdfs://localhost:44999/hbase/data/default/testReplayEditsAfterAbortingFlush/6c4b2f0afc1fab59a8a3937aad627963/recovered.edits/0000000000000000004-wal.1732631488140.temp (wrote 20 edits, skipped 0 edits in 0 ms) at 1732631497319 (+10 ms)Rename recovered edits hdfs://localhost:44999/hbase/data/default/testReplayEditsAfterAbortingFlush/6c4b2f0afc1fab59a8a3937aad627963/recovered.edits/0000000000000000004-wal.1732631488140.temp to hdfs://localhost:44999/hbase/data/default/testReplayEditsAfterAbortingFlush/6c4b2f0afc1fab59a8a3937aad627963/recovered.edits/0000000000000000026 at 1732631497320 (+1 ms)Processed 23 edits across 1 Regions in 16 ms; skipped=3; WAL=hdfs://localhost:44999/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1732631488070/wal.1732631488140, size=2.3 K, length=2398, corrupted=false, cancelled=false at 1732631497320 2024-11-26T14:31:37,322 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:44999/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1732631488070/wal.1732631488140 to hdfs://localhost:44999/hbase/oldWALs/wal.1732631488140 2024-11-26T14:31:37,323 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:44999/hbase/data/default/testReplayEditsAfterAbortingFlush/6c4b2f0afc1fab59a8a3937aad627963/recovered.edits/0000000000000000026 2024-11-26T14:31:37,323 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-26T14:31:37,324 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:44999/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1732631488070, archiveDir=hdfs://localhost:44999/hbase/oldWALs, maxLogs=32 2024-11-26T14:31:37,336 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1732631488070/wal.1732631497325, exclude list is [], retry=0 2024-11-26T14:31:37,339 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38147,DS-44b12d44-38d0-4c76-b56f-3ca8d8caa398,DISK] 2024-11-26T14:31:37,339 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36277,DS-9cb9383d-a6c8-4ba6-83b3-6706331a6539,DISK] 2024-11-26T14:31:37,340 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36307,DS-dd37de78-b3d5-4d7c-8223-4b15101c21bf,DISK] 2024-11-26T14:31:37,341 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1732631488070/wal.1732631497325 2024-11-26T14:31:37,342 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:35081:35081),(127.0.0.1/127.0.0.1:40449:40449),(127.0.0.1/127.0.0.1:34431:34431)] 2024-11-26T14:31:37,342 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 6c4b2f0afc1fab59a8a3937aad627963, NAME => 'testReplayEditsAfterAbortingFlush,,1732631488072.6c4b2f0afc1fab59a8a3937aad627963.', STARTKEY => '', ENDKEY => ''} 2024-11-26T14:31:37,343 DEBUG [Time-limited test {}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testReplayEditsAfterAbortingFlush 6c4b2f0afc1fab59a8a3937aad627963 2024-11-26T14:31:37,343 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsAfterAbortingFlush,,1732631488072.6c4b2f0afc1fab59a8a3937aad627963.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-26T14:31:37,343 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 6c4b2f0afc1fab59a8a3937aad627963 2024-11-26T14:31:37,343 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 6c4b2f0afc1fab59a8a3937aad627963 2024-11-26T14:31:37,345 INFO [StoreOpener-6c4b2f0afc1fab59a8a3937aad627963-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 6c4b2f0afc1fab59a8a3937aad627963 2024-11-26T14:31:37,346 INFO [StoreOpener-6c4b2f0afc1fab59a8a3937aad627963-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 6c4b2f0afc1fab59a8a3937aad627963 columnFamilyName a 2024-11-26T14:31:37,346 DEBUG [StoreOpener-6c4b2f0afc1fab59a8a3937aad627963-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:31:37,346 INFO [StoreOpener-6c4b2f0afc1fab59a8a3937aad627963-1 {}] regionserver.HStore(327): Store=6c4b2f0afc1fab59a8a3937aad627963/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-26T14:31:37,347 INFO [StoreOpener-6c4b2f0afc1fab59a8a3937aad627963-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 6c4b2f0afc1fab59a8a3937aad627963 2024-11-26T14:31:37,347 INFO [StoreOpener-6c4b2f0afc1fab59a8a3937aad627963-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 6c4b2f0afc1fab59a8a3937aad627963 columnFamilyName b 2024-11-26T14:31:37,347 DEBUG [StoreOpener-6c4b2f0afc1fab59a8a3937aad627963-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:31:37,348 INFO [StoreOpener-6c4b2f0afc1fab59a8a3937aad627963-1 {}] regionserver.HStore(327): Store=6c4b2f0afc1fab59a8a3937aad627963/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-26T14:31:37,348 INFO [StoreOpener-6c4b2f0afc1fab59a8a3937aad627963-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 6c4b2f0afc1fab59a8a3937aad627963 2024-11-26T14:31:37,349 INFO [StoreOpener-6c4b2f0afc1fab59a8a3937aad627963-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 6c4b2f0afc1fab59a8a3937aad627963 columnFamilyName c 2024-11-26T14:31:37,349 DEBUG [StoreOpener-6c4b2f0afc1fab59a8a3937aad627963-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:31:37,350 INFO [StoreOpener-6c4b2f0afc1fab59a8a3937aad627963-1 {}] regionserver.HStore(327): Store=6c4b2f0afc1fab59a8a3937aad627963/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-26T14:31:37,350 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 6c4b2f0afc1fab59a8a3937aad627963 2024-11-26T14:31:37,350 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44999/hbase/data/default/testReplayEditsAfterAbortingFlush/6c4b2f0afc1fab59a8a3937aad627963 2024-11-26T14:31:37,352 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:44999/hbase/data/default/testReplayEditsAfterAbortingFlush/6c4b2f0afc1fab59a8a3937aad627963 2024-11-26T14:31:37,353 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:44999/hbase/data/default/testReplayEditsAfterAbortingFlush/6c4b2f0afc1fab59a8a3937aad627963/recovered.edits/0000000000000000026 2024-11-26T14:31:37,356 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:44999/hbase/data/default/testReplayEditsAfterAbortingFlush/6c4b2f0afc1fab59a8a3937aad627963/recovered.edits/0000000000000000026: isRecoveredEdits=true, hasTagCompression=true, hasValueCompression=false, valueCompressionType=null 2024-11-26T14:31:37,357 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 20, skipped 0, firstSequenceIdInLog=4, maxSequenceIdInLog=26, path=hdfs://localhost:44999/hbase/data/default/testReplayEditsAfterAbortingFlush/6c4b2f0afc1fab59a8a3937aad627963/recovered.edits/0000000000000000026 2024-11-26T14:31:37,358 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 6c4b2f0afc1fab59a8a3937aad627963 3/3 column families, dataSize=1.16 KB heapSize=3.41 KB 2024-11-26T14:31:37,378 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44999/hbase/data/default/testReplayEditsAfterAbortingFlush/6c4b2f0afc1fab59a8a3937aad627963/.tmp/a/56a8de61451e4cc382249d5a7df1ccb7 is 64, key is testReplayEditsAfterAbortingFlush12/a:q/1732631497272/Put/seqid=0 2024-11-26T14:31:37,385 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36277 is added to blk_1073741906_1084 (size=5523) 2024-11-26T14:31:37,385 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38147 is added to blk_1073741906_1084 (size=5523) 2024-11-26T14:31:37,386 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36307 is added to blk_1073741906_1084 (size=5523) 2024-11-26T14:31:37,387 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=416 B at sequenceid=26 (bloomFilter=true), to=hdfs://localhost:44999/hbase/data/default/testReplayEditsAfterAbortingFlush/6c4b2f0afc1fab59a8a3937aad627963/.tmp/a/56a8de61451e4cc382249d5a7df1ccb7 2024-11-26T14:31:37,407 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44999/hbase/data/default/testReplayEditsAfterAbortingFlush/6c4b2f0afc1fab59a8a3937aad627963/.tmp/b/c8245a1754a44268a72e1e27a08f758f is 64, key is testReplayEditsAfterAbortingFlush10/b:q/1732631497262/Put/seqid=0 2024-11-26T14:31:37,413 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36277 is added to blk_1073741907_1085 (size=5524) 2024-11-26T14:31:37,414 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36307 is added to blk_1073741907_1085 (size=5524) 2024-11-26T14:31:37,414 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38147 is added to blk_1073741907_1085 (size=5524) 2024-11-26T14:31:37,415 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=417 B at sequenceid=26 (bloomFilter=true), to=hdfs://localhost:44999/hbase/data/default/testReplayEditsAfterAbortingFlush/6c4b2f0afc1fab59a8a3937aad627963/.tmp/b/c8245a1754a44268a72e1e27a08f758f 2024-11-26T14:31:37,433 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44999/hbase/data/default/testReplayEditsAfterAbortingFlush/6c4b2f0afc1fab59a8a3937aad627963/.tmp/c/115fc14e48fd45659987c0eafbc4d049 is 64, key is testReplayEditsAfterAbortingFlush11/c:q/1732631497268/Put/seqid=0 2024-11-26T14:31:37,439 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36277 is added to blk_1073741908_1086 (size=5457) 2024-11-26T14:31:37,440 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36307 is added to blk_1073741908_1086 (size=5457) 2024-11-26T14:31:37,440 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38147 is added to blk_1073741908_1086 (size=5457) 2024-11-26T14:31:37,440 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=357 B at sequenceid=26 (bloomFilter=true), to=hdfs://localhost:44999/hbase/data/default/testReplayEditsAfterAbortingFlush/6c4b2f0afc1fab59a8a3937aad627963/.tmp/c/115fc14e48fd45659987c0eafbc4d049 2024-11-26T14:31:37,447 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44999/hbase/data/default/testReplayEditsAfterAbortingFlush/6c4b2f0afc1fab59a8a3937aad627963/.tmp/a/56a8de61451e4cc382249d5a7df1ccb7 as hdfs://localhost:44999/hbase/data/default/testReplayEditsAfterAbortingFlush/6c4b2f0afc1fab59a8a3937aad627963/a/56a8de61451e4cc382249d5a7df1ccb7 2024-11-26T14:31:37,453 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44999/hbase/data/default/testReplayEditsAfterAbortingFlush/6c4b2f0afc1fab59a8a3937aad627963/a/56a8de61451e4cc382249d5a7df1ccb7, entries=7, sequenceid=26, filesize=5.4 K 2024-11-26T14:31:37,454 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44999/hbase/data/default/testReplayEditsAfterAbortingFlush/6c4b2f0afc1fab59a8a3937aad627963/.tmp/b/c8245a1754a44268a72e1e27a08f758f as hdfs://localhost:44999/hbase/data/default/testReplayEditsAfterAbortingFlush/6c4b2f0afc1fab59a8a3937aad627963/b/c8245a1754a44268a72e1e27a08f758f 2024-11-26T14:31:37,460 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44999/hbase/data/default/testReplayEditsAfterAbortingFlush/6c4b2f0afc1fab59a8a3937aad627963/b/c8245a1754a44268a72e1e27a08f758f, entries=7, sequenceid=26, filesize=5.4 K 2024-11-26T14:31:37,461 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44999/hbase/data/default/testReplayEditsAfterAbortingFlush/6c4b2f0afc1fab59a8a3937aad627963/.tmp/c/115fc14e48fd45659987c0eafbc4d049 as hdfs://localhost:44999/hbase/data/default/testReplayEditsAfterAbortingFlush/6c4b2f0afc1fab59a8a3937aad627963/c/115fc14e48fd45659987c0eafbc4d049 2024-11-26T14:31:37,467 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44999/hbase/data/default/testReplayEditsAfterAbortingFlush/6c4b2f0afc1fab59a8a3937aad627963/c/115fc14e48fd45659987c0eafbc4d049, entries=6, sequenceid=26, filesize=5.3 K 2024-11-26T14:31:37,467 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~1.16 KB/1190, heapSize ~3.36 KB/3440, currentSize=0 B/0 for 6c4b2f0afc1fab59a8a3937aad627963 in 109ms, sequenceid=26, compaction requested=false; wal=null 2024-11-26T14:31:37,468 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:44999/hbase/data/default/testReplayEditsAfterAbortingFlush/6c4b2f0afc1fab59a8a3937aad627963/recovered.edits/0000000000000000026 2024-11-26T14:31:37,469 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 6c4b2f0afc1fab59a8a3937aad627963 2024-11-26T14:31:37,469 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 6c4b2f0afc1fab59a8a3937aad627963 2024-11-26T14:31:37,470 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsAfterAbortingFlush descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-26T14:31:37,471 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 6c4b2f0afc1fab59a8a3937aad627963 2024-11-26T14:31:37,473 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44999/hbase/data/default/testReplayEditsAfterAbortingFlush/6c4b2f0afc1fab59a8a3937aad627963/recovered.edits/26.seqid, newMaxSeqId=26, maxSeqId=1 2024-11-26T14:31:37,474 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 6c4b2f0afc1fab59a8a3937aad627963; next sequenceid=27; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=59603568, jitterRate=-0.11183762550354004}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-26T14:31:37,474 DEBUG [Time-limited test {}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 6c4b2f0afc1fab59a8a3937aad627963 2024-11-26T14:31:37,475 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 6c4b2f0afc1fab59a8a3937aad627963: Running coprocessor pre-open hook at 1732631497343Writing region info on filesystem at 1732631497343Initializing all the Stores at 1732631497344 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732631497344Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732631497344Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732631497344Obtaining lock to block concurrent updates at 1732631497358 (+14 ms)Preparing flush snapshotting stores in 6c4b2f0afc1fab59a8a3937aad627963 at 1732631497358Finished memstore snapshotting testReplayEditsAfterAbortingFlush,,1732631488072.6c4b2f0afc1fab59a8a3937aad627963., syncing WAL and waiting on mvcc, flushsize=dataSize=1190, getHeapSize=3440, getOffHeapSize=0, getCellsCount=20 at 1732631497358Flushing stores of testReplayEditsAfterAbortingFlush,,1732631488072.6c4b2f0afc1fab59a8a3937aad627963. at 1732631497358Flushing 6c4b2f0afc1fab59a8a3937aad627963/a: creating writer at 1732631497358Flushing 6c4b2f0afc1fab59a8a3937aad627963/a: appending metadata at 1732631497378 (+20 ms)Flushing 6c4b2f0afc1fab59a8a3937aad627963/a: closing flushed file at 1732631497378Flushing 6c4b2f0afc1fab59a8a3937aad627963/b: creating writer at 1732631497393 (+15 ms)Flushing 6c4b2f0afc1fab59a8a3937aad627963/b: appending metadata at 1732631497407 (+14 ms)Flushing 6c4b2f0afc1fab59a8a3937aad627963/b: closing flushed file at 1732631497407Flushing 6c4b2f0afc1fab59a8a3937aad627963/c: creating writer at 1732631497420 (+13 ms)Flushing 6c4b2f0afc1fab59a8a3937aad627963/c: appending metadata at 1732631497433 (+13 ms)Flushing 6c4b2f0afc1fab59a8a3937aad627963/c: closing flushed file at 1732631497433Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@510ec3f2: reopening flushed file at 1732631497446 (+13 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@18fb5388: reopening flushed file at 1732631497453 (+7 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@425deefc: reopening flushed file at 1732631497460 (+7 ms)Finished flush of dataSize ~1.16 KB/1190, heapSize ~3.36 KB/3440, currentSize=0 B/0 for 6c4b2f0afc1fab59a8a3937aad627963 in 109ms, sequenceid=26, compaction requested=false; wal=null at 1732631497467 (+7 ms)Cleaning up temporary data from old regions at 1732631497469 (+2 ms)Running coprocessor post-open hooks at 1732631497474 (+5 ms)Region opened successfully at 1732631497475 (+1 ms) 2024-11-26T14:31:37,497 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestAsyncWALReplayCompressed#testReplayEditsAfterAbortingFlush Thread=423 (was 424), OpenFileDescriptor=1183 (was 1127) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=239 (was 260), ProcessCount=11 (was 11), AvailableMemoryMB=8990 (was 9029) 2024-11-26T14:31:37,497 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1183 is superior to 1024 2024-11-26T14:31:37,510 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestAsyncWALReplayCompressed#testDatalossWhenInputError Thread=423, OpenFileDescriptor=1183, MaxFileDescriptor=1048576, SystemLoadAverage=239, ProcessCount=11, AvailableMemoryMB=8989 2024-11-26T14:31:37,510 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1183 is superior to 1024 2024-11-26T14:31:37,524 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-26T14:31:37,527 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-26T14:31:37,528 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-26T14:31:37,531 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-34123990, suffix=, logDir=hdfs://localhost:44999/hbase/WALs/hregion-34123990, archiveDir=hdfs://localhost:44999/hbase/oldWALs, maxLogs=32 2024-11-26T14:31:37,545 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-34123990/hregion-34123990.1732631497531, exclude list is [], retry=0 2024-11-26T14:31:37,548 DEBUG [AsyncFSWAL-24-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36307,DS-dd37de78-b3d5-4d7c-8223-4b15101c21bf,DISK] 2024-11-26T14:31:37,548 DEBUG [AsyncFSWAL-24-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36277,DS-9cb9383d-a6c8-4ba6-83b3-6706331a6539,DISK] 2024-11-26T14:31:37,548 DEBUG [AsyncFSWAL-24-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38147,DS-44b12d44-38d0-4c76-b56f-3ca8d8caa398,DISK] 2024-11-26T14:31:37,550 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-34123990/hregion-34123990.1732631497531 2024-11-26T14:31:37,551 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:34431:34431),(127.0.0.1/127.0.0.1:40449:40449),(127.0.0.1/127.0.0.1:35081:35081)] 2024-11-26T14:31:37,551 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => f70d4c68e60cf6ec51a282d0cc77a081, NAME => 'testDatalossWhenInputError,,1732631497525.f70d4c68e60cf6ec51a282d0cc77a081.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testDatalossWhenInputError', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:44999/hbase 2024-11-26T14:31:37,563 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38147 is added to blk_1073741910_1088 (size=61) 2024-11-26T14:31:37,564 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36307 is added to blk_1073741910_1088 (size=61) 2024-11-26T14:31:37,564 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36277 is added to blk_1073741910_1088 (size=61) 2024-11-26T14:31:37,564 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testDatalossWhenInputError,,1732631497525.f70d4c68e60cf6ec51a282d0cc77a081.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-26T14:31:37,566 INFO [StoreOpener-f70d4c68e60cf6ec51a282d0cc77a081-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region f70d4c68e60cf6ec51a282d0cc77a081 2024-11-26T14:31:37,567 INFO [StoreOpener-f70d4c68e60cf6ec51a282d0cc77a081-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region f70d4c68e60cf6ec51a282d0cc77a081 columnFamilyName a 2024-11-26T14:31:37,567 DEBUG [StoreOpener-f70d4c68e60cf6ec51a282d0cc77a081-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:31:37,568 INFO [StoreOpener-f70d4c68e60cf6ec51a282d0cc77a081-1 {}] regionserver.HStore(327): Store=f70d4c68e60cf6ec51a282d0cc77a081/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-26T14:31:37,568 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for f70d4c68e60cf6ec51a282d0cc77a081 2024-11-26T14:31:37,569 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44999/hbase/data/default/testDatalossWhenInputError/f70d4c68e60cf6ec51a282d0cc77a081 2024-11-26T14:31:37,569 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44999/hbase/data/default/testDatalossWhenInputError/f70d4c68e60cf6ec51a282d0cc77a081 2024-11-26T14:31:37,570 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for f70d4c68e60cf6ec51a282d0cc77a081 2024-11-26T14:31:37,570 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for f70d4c68e60cf6ec51a282d0cc77a081 2024-11-26T14:31:37,571 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for f70d4c68e60cf6ec51a282d0cc77a081 2024-11-26T14:31:37,573 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44999/hbase/data/default/testDatalossWhenInputError/f70d4c68e60cf6ec51a282d0cc77a081/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-26T14:31:37,574 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened f70d4c68e60cf6ec51a282d0cc77a081; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=70756546, jitterRate=0.054354697465896606}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-26T14:31:37,574 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for f70d4c68e60cf6ec51a282d0cc77a081: Writing region info on filesystem at 1732631497564Initializing all the Stores at 1732631497565 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732631497565Cleaning up temporary data from old regions at 1732631497570 (+5 ms)Region opened successfully at 1732631497574 (+4 ms) 2024-11-26T14:31:37,575 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing f70d4c68e60cf6ec51a282d0cc77a081, disabling compactions & flushes 2024-11-26T14:31:37,575 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testDatalossWhenInputError,,1732631497525.f70d4c68e60cf6ec51a282d0cc77a081. 2024-11-26T14:31:37,575 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testDatalossWhenInputError,,1732631497525.f70d4c68e60cf6ec51a282d0cc77a081. 2024-11-26T14:31:37,575 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testDatalossWhenInputError,,1732631497525.f70d4c68e60cf6ec51a282d0cc77a081. after waiting 0 ms 2024-11-26T14:31:37,575 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testDatalossWhenInputError,,1732631497525.f70d4c68e60cf6ec51a282d0cc77a081. 2024-11-26T14:31:37,575 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testDatalossWhenInputError,,1732631497525.f70d4c68e60cf6ec51a282d0cc77a081. 2024-11-26T14:31:37,575 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for f70d4c68e60cf6ec51a282d0cc77a081: Waiting for close lock at 1732631497574Disabling compacts and flushes for region at 1732631497574Disabling writes for close at 1732631497575 (+1 ms)Writing region close event to WAL at 1732631497575Closed at 1732631497575 2024-11-26T14:31:37,580 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36277 is added to blk_1073741909_1087 (size=93) 2024-11-26T14:31:37,580 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36307 is added to blk_1073741909_1087 (size=93) 2024-11-26T14:31:37,580 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38147 is added to blk_1073741909_1087 (size=93) 2024-11-26T14:31:37,582 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-11-26T14:31:37,583 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-34123990:(num 1732631497531) 2024-11-26T14:31:37,583 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-26T14:31:37,584 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:44999/hbase/WALs/testdatalosswheninputerror-manual,16010,1732631497524, archiveDir=hdfs://localhost:44999/hbase/oldWALs, maxLogs=32 2024-11-26T14:31:37,596 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testdatalosswheninputerror-manual,16010,1732631497524/wal.1732631497585, exclude list is [], retry=0 2024-11-26T14:31:37,599 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36307,DS-dd37de78-b3d5-4d7c-8223-4b15101c21bf,DISK] 2024-11-26T14:31:37,599 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36277,DS-9cb9383d-a6c8-4ba6-83b3-6706331a6539,DISK] 2024-11-26T14:31:37,599 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38147,DS-44b12d44-38d0-4c76-b56f-3ca8d8caa398,DISK] 2024-11-26T14:31:37,601 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testdatalosswheninputerror-manual,16010,1732631497524/wal.1732631497585 2024-11-26T14:31:37,601 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:34431:34431),(127.0.0.1/127.0.0.1:40449:40449),(127.0.0.1/127.0.0.1:35081:35081)] 2024-11-26T14:31:37,601 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => f70d4c68e60cf6ec51a282d0cc77a081, NAME => 'testDatalossWhenInputError,,1732631497525.f70d4c68e60cf6ec51a282d0cc77a081.', STARTKEY => '', ENDKEY => ''} 2024-11-26T14:31:37,601 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testDatalossWhenInputError,,1732631497525.f70d4c68e60cf6ec51a282d0cc77a081.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-26T14:31:37,601 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for f70d4c68e60cf6ec51a282d0cc77a081 2024-11-26T14:31:37,602 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for f70d4c68e60cf6ec51a282d0cc77a081 2024-11-26T14:31:37,603 INFO [StoreOpener-f70d4c68e60cf6ec51a282d0cc77a081-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region f70d4c68e60cf6ec51a282d0cc77a081 2024-11-26T14:31:37,604 INFO [StoreOpener-f70d4c68e60cf6ec51a282d0cc77a081-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region f70d4c68e60cf6ec51a282d0cc77a081 columnFamilyName a 2024-11-26T14:31:37,605 DEBUG [StoreOpener-f70d4c68e60cf6ec51a282d0cc77a081-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:31:37,605 INFO [StoreOpener-f70d4c68e60cf6ec51a282d0cc77a081-1 {}] regionserver.HStore(327): Store=f70d4c68e60cf6ec51a282d0cc77a081/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-26T14:31:37,605 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for f70d4c68e60cf6ec51a282d0cc77a081 2024-11-26T14:31:37,606 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44999/hbase/data/default/testDatalossWhenInputError/f70d4c68e60cf6ec51a282d0cc77a081 2024-11-26T14:31:37,607 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44999/hbase/data/default/testDatalossWhenInputError/f70d4c68e60cf6ec51a282d0cc77a081 2024-11-26T14:31:37,607 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for f70d4c68e60cf6ec51a282d0cc77a081 2024-11-26T14:31:37,607 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for f70d4c68e60cf6ec51a282d0cc77a081 2024-11-26T14:31:37,609 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for f70d4c68e60cf6ec51a282d0cc77a081 2024-11-26T14:31:37,610 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened f70d4c68e60cf6ec51a282d0cc77a081; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=70144310, jitterRate=0.04523167014122009}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-26T14:31:37,611 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for f70d4c68e60cf6ec51a282d0cc77a081: Writing region info on filesystem at 1732631497602Initializing all the Stores at 1732631497602Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732631497602Cleaning up temporary data from old regions at 1732631497607 (+5 ms)Region opened successfully at 1732631497611 (+4 ms) 2024-11-26T14:31:37,618 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing f70d4c68e60cf6ec51a282d0cc77a081, disabling compactions & flushes 2024-11-26T14:31:37,618 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testDatalossWhenInputError,,1732631497525.f70d4c68e60cf6ec51a282d0cc77a081. 2024-11-26T14:31:37,618 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testDatalossWhenInputError,,1732631497525.f70d4c68e60cf6ec51a282d0cc77a081. 2024-11-26T14:31:37,618 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testDatalossWhenInputError,,1732631497525.f70d4c68e60cf6ec51a282d0cc77a081. after waiting 0 ms 2024-11-26T14:31:37,618 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testDatalossWhenInputError,,1732631497525.f70d4c68e60cf6ec51a282d0cc77a081. 2024-11-26T14:31:37,618 ERROR [Time-limited test {}] regionserver.HRegion(1960): Memstore data size is 750 in region testDatalossWhenInputError,,1732631497525.f70d4c68e60cf6ec51a282d0cc77a081. 2024-11-26T14:31:37,619 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testDatalossWhenInputError,,1732631497525.f70d4c68e60cf6ec51a282d0cc77a081. 2024-11-26T14:31:37,619 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for f70d4c68e60cf6ec51a282d0cc77a081: Waiting for close lock at 1732631497618Disabling compacts and flushes for region at 1732631497618Disabling writes for close at 1732631497618Writing region close event to WAL at 1732631497618Closed at 1732631497619 (+1 ms) 2024-11-26T14:31:37,621 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36307 is added to blk_1073741911_1089 (size=838) 2024-11-26T14:31:37,621 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38147 is added to blk_1073741911_1089 (size=838) 2024-11-26T14:31:37,621 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36277 is added to blk_1073741911_1089 (size=838) 2024-11-26T14:31:37,634 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:44999/hbase/WALs/testdatalosswheninputerror-manual,16010,1732631497524/wal.1732631497585, size=838 (838bytes) 2024-11-26T14:31:37,634 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:44999/hbase/WALs/testdatalosswheninputerror-manual,16010,1732631497524/wal.1732631497585 2024-11-26T14:31:37,635 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:44999/hbase/WALs/testdatalosswheninputerror-manual,16010,1732631497524/wal.1732631497585 after 0ms 2024-11-26T14:31:37,637 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:44999/hbase/WALs/testdatalosswheninputerror-manual,16010,1732631497524/wal.1732631497585: isRecoveredEdits=false, hasTagCompression=true, hasValueCompression=false, valueCompressionType=null 2024-11-26T14:31:37,637 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:44999/hbase/WALs/testdatalosswheninputerror-manual,16010,1732631497524/wal.1732631497585 took 3ms 2024-11-26T14:31:37,638 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:44999/hbase/WALs/testdatalosswheninputerror-manual,16010,1732631497524/wal.1732631497585 so closing down 2024-11-26T14:31:37,638 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-26T14:31:37,640 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000003-wal.1732631497585.temp 2024-11-26T14:31:37,641 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:44999/hbase/data/default/testDatalossWhenInputError/f70d4c68e60cf6ec51a282d0cc77a081/recovered.edits/0000000000000000003-wal.1732631497585.temp 2024-11-26T14:31:37,641 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-11-26T14:31:37,647 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36277 is added to blk_1073741912_1090 (size=838) 2024-11-26T14:31:37,647 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36307 is added to blk_1073741912_1090 (size=838) 2024-11-26T14:31:37,647 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38147 is added to blk_1073741912_1090 (size=838) 2024-11-26T14:31:37,648 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:44999/hbase/data/default/testDatalossWhenInputError/f70d4c68e60cf6ec51a282d0cc77a081/recovered.edits/0000000000000000003-wal.1732631497585.temp (wrote 10 edits, skipped 0 edits in 0 ms) 2024-11-26T14:31:37,649 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:44999/hbase/data/default/testDatalossWhenInputError/f70d4c68e60cf6ec51a282d0cc77a081/recovered.edits/0000000000000000003-wal.1732631497585.temp to hdfs://localhost:44999/hbase/data/default/testDatalossWhenInputError/f70d4c68e60cf6ec51a282d0cc77a081/recovered.edits/0000000000000000012 2024-11-26T14:31:37,649 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 10 edits across 1 Regions in 12 ms; skipped=0; WAL=hdfs://localhost:44999/hbase/WALs/testdatalosswheninputerror-manual,16010,1732631497524/wal.1732631497585, size=838, length=838, corrupted=false, cancelled=false 2024-11-26T14:31:37,649 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:44999/hbase/WALs/testdatalosswheninputerror-manual,16010,1732631497524/wal.1732631497585, journal: Splitting hdfs://localhost:44999/hbase/WALs/testdatalosswheninputerror-manual,16010,1732631497524/wal.1732631497585, size=838 (838bytes) at 1732631497634Finishing writing output for hdfs://localhost:44999/hbase/WALs/testdatalosswheninputerror-manual,16010,1732631497524/wal.1732631497585 so closing down at 1732631497638 (+4 ms)Creating recovered edits writer path=hdfs://localhost:44999/hbase/data/default/testDatalossWhenInputError/f70d4c68e60cf6ec51a282d0cc77a081/recovered.edits/0000000000000000003-wal.1732631497585.temp at 1732631497641 (+3 ms)3 split writer threads finished at 1732631497641Closed recovered edits writer path=hdfs://localhost:44999/hbase/data/default/testDatalossWhenInputError/f70d4c68e60cf6ec51a282d0cc77a081/recovered.edits/0000000000000000003-wal.1732631497585.temp (wrote 10 edits, skipped 0 edits in 0 ms) at 1732631497648 (+7 ms)Rename recovered edits hdfs://localhost:44999/hbase/data/default/testDatalossWhenInputError/f70d4c68e60cf6ec51a282d0cc77a081/recovered.edits/0000000000000000003-wal.1732631497585.temp to hdfs://localhost:44999/hbase/data/default/testDatalossWhenInputError/f70d4c68e60cf6ec51a282d0cc77a081/recovered.edits/0000000000000000012 at 1732631497649 (+1 ms)Processed 10 edits across 1 Regions in 12 ms; skipped=0; WAL=hdfs://localhost:44999/hbase/WALs/testdatalosswheninputerror-manual,16010,1732631497524/wal.1732631497585, size=838, length=838, corrupted=false, cancelled=false at 1732631497649 2024-11-26T14:31:37,651 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:44999/hbase/WALs/testdatalosswheninputerror-manual,16010,1732631497524/wal.1732631497585 to hdfs://localhost:44999/hbase/oldWALs/wal.1732631497585 2024-11-26T14:31:37,652 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:44999/hbase/data/default/testDatalossWhenInputError/f70d4c68e60cf6ec51a282d0cc77a081/recovered.edits/0000000000000000012 2024-11-26T14:31:37,656 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:44999/hbase/data/default/testDatalossWhenInputError/f70d4c68e60cf6ec51a282d0cc77a081/recovered.edits/0000000000000000012: isRecoveredEdits=true, hasTagCompression=true, hasValueCompression=false, valueCompressionType=null 2024-11-26T14:31:37,881 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-26T14:31:38,019 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-26T14:31:38,022 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:44999/hbase/WALs/testdatalosswheninputerror-manual,16010,1732631497524, archiveDir=hdfs://localhost:44999/hbase/oldWALs, maxLogs=32 2024-11-26T14:31:38,033 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testdatalosswheninputerror-manual,16010,1732631497524/wal.1732631498022, exclude list is [], retry=0 2024-11-26T14:31:38,036 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36307,DS-dd37de78-b3d5-4d7c-8223-4b15101c21bf,DISK] 2024-11-26T14:31:38,036 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38147,DS-44b12d44-38d0-4c76-b56f-3ca8d8caa398,DISK] 2024-11-26T14:31:38,036 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36277,DS-9cb9383d-a6c8-4ba6-83b3-6706331a6539,DISK] 2024-11-26T14:31:38,038 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testdatalosswheninputerror-manual,16010,1732631497524/wal.1732631498022 2024-11-26T14:31:38,038 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:34431:34431),(127.0.0.1/127.0.0.1:35081:35081),(127.0.0.1/127.0.0.1:40449:40449)] 2024-11-26T14:31:38,038 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => f70d4c68e60cf6ec51a282d0cc77a081, NAME => 'testDatalossWhenInputError,,1732631497525.f70d4c68e60cf6ec51a282d0cc77a081.', STARTKEY => '', ENDKEY => ''} 2024-11-26T14:31:38,038 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testDatalossWhenInputError,,1732631497525.f70d4c68e60cf6ec51a282d0cc77a081.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-26T14:31:38,038 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for f70d4c68e60cf6ec51a282d0cc77a081 2024-11-26T14:31:38,038 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for f70d4c68e60cf6ec51a282d0cc77a081 2024-11-26T14:31:38,042 INFO [StoreOpener-f70d4c68e60cf6ec51a282d0cc77a081-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region f70d4c68e60cf6ec51a282d0cc77a081 2024-11-26T14:31:38,044 INFO [StoreOpener-f70d4c68e60cf6ec51a282d0cc77a081-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region f70d4c68e60cf6ec51a282d0cc77a081 columnFamilyName a 2024-11-26T14:31:38,044 DEBUG [StoreOpener-f70d4c68e60cf6ec51a282d0cc77a081-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:31:38,045 INFO [StoreOpener-f70d4c68e60cf6ec51a282d0cc77a081-1 {}] regionserver.HStore(327): Store=f70d4c68e60cf6ec51a282d0cc77a081/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-26T14:31:38,045 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for f70d4c68e60cf6ec51a282d0cc77a081 2024-11-26T14:31:38,046 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44999/hbase/data/default/testDatalossWhenInputError/f70d4c68e60cf6ec51a282d0cc77a081 2024-11-26T14:31:38,048 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:44999/hbase/data/default/testDatalossWhenInputError/f70d4c68e60cf6ec51a282d0cc77a081 2024-11-26T14:31:38,048 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:44999/hbase/data/default/testDatalossWhenInputError/f70d4c68e60cf6ec51a282d0cc77a081/recovered.edits/0000000000000000012 2024-11-26T14:31:38,050 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:44999/hbase/data/default/testDatalossWhenInputError/f70d4c68e60cf6ec51a282d0cc77a081/recovered.edits/0000000000000000012: isRecoveredEdits=true, hasTagCompression=true, hasValueCompression=false, valueCompressionType=null 2024-11-26T14:31:38,051 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 10, skipped 0, firstSequenceIdInLog=3, maxSequenceIdInLog=12, path=hdfs://localhost:44999/hbase/data/default/testDatalossWhenInputError/f70d4c68e60cf6ec51a282d0cc77a081/recovered.edits/0000000000000000012 2024-11-26T14:31:38,051 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing f70d4c68e60cf6ec51a282d0cc77a081 1/1 column families, dataSize=750 B heapSize=1.73 KB 2024-11-26T14:31:38,069 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44999/hbase/data/default/testDatalossWhenInputError/f70d4c68e60cf6ec51a282d0cc77a081/.tmp/a/8902e370da5b458db77ec6430f459b6a is 79, key is testDatalossWhenInputError/a:x0/1732631497611/Put/seqid=0 2024-11-26T14:31:38,076 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38147 is added to blk_1073741914_1092 (size=5808) 2024-11-26T14:31:38,076 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36277 is added to blk_1073741914_1092 (size=5808) 2024-11-26T14:31:38,076 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36307 is added to blk_1073741914_1092 (size=5808) 2024-11-26T14:31:38,077 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=750 B at sequenceid=12 (bloomFilter=true), to=hdfs://localhost:44999/hbase/data/default/testDatalossWhenInputError/f70d4c68e60cf6ec51a282d0cc77a081/.tmp/a/8902e370da5b458db77ec6430f459b6a 2024-11-26T14:31:38,087 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44999/hbase/data/default/testDatalossWhenInputError/f70d4c68e60cf6ec51a282d0cc77a081/.tmp/a/8902e370da5b458db77ec6430f459b6a as hdfs://localhost:44999/hbase/data/default/testDatalossWhenInputError/f70d4c68e60cf6ec51a282d0cc77a081/a/8902e370da5b458db77ec6430f459b6a 2024-11-26T14:31:38,096 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44999/hbase/data/default/testDatalossWhenInputError/f70d4c68e60cf6ec51a282d0cc77a081/a/8902e370da5b458db77ec6430f459b6a, entries=10, sequenceid=12, filesize=5.7 K 2024-11-26T14:31:38,097 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~750 B/750, heapSize ~1.72 KB/1760, currentSize=0 B/0 for f70d4c68e60cf6ec51a282d0cc77a081 in 46ms, sequenceid=12, compaction requested=false; wal=null 2024-11-26T14:31:38,097 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:44999/hbase/data/default/testDatalossWhenInputError/f70d4c68e60cf6ec51a282d0cc77a081/recovered.edits/0000000000000000012 2024-11-26T14:31:38,098 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for f70d4c68e60cf6ec51a282d0cc77a081 2024-11-26T14:31:38,098 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for f70d4c68e60cf6ec51a282d0cc77a081 2024-11-26T14:31:38,102 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for f70d4c68e60cf6ec51a282d0cc77a081 2024-11-26T14:31:38,104 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44999/hbase/data/default/testDatalossWhenInputError/f70d4c68e60cf6ec51a282d0cc77a081/recovered.edits/12.seqid, newMaxSeqId=12, maxSeqId=1 2024-11-26T14:31:38,105 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened f70d4c68e60cf6ec51a282d0cc77a081; next sequenceid=13; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=69822922, jitterRate=0.04044261574745178}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-26T14:31:38,106 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for f70d4c68e60cf6ec51a282d0cc77a081: Writing region info on filesystem at 1732631498039Initializing all the Stores at 1732631498041 (+2 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732631498041Obtaining lock to block concurrent updates at 1732631498052 (+11 ms)Preparing flush snapshotting stores in f70d4c68e60cf6ec51a282d0cc77a081 at 1732631498052Finished memstore snapshotting testDatalossWhenInputError,,1732631497525.f70d4c68e60cf6ec51a282d0cc77a081., syncing WAL and waiting on mvcc, flushsize=dataSize=750, getHeapSize=1760, getOffHeapSize=0, getCellsCount=10 at 1732631498052Flushing stores of testDatalossWhenInputError,,1732631497525.f70d4c68e60cf6ec51a282d0cc77a081. at 1732631498052Flushing f70d4c68e60cf6ec51a282d0cc77a081/a: creating writer at 1732631498052Flushing f70d4c68e60cf6ec51a282d0cc77a081/a: appending metadata at 1732631498069 (+17 ms)Flushing f70d4c68e60cf6ec51a282d0cc77a081/a: closing flushed file at 1732631498069Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@2260eab2: reopening flushed file at 1732631498085 (+16 ms)Finished flush of dataSize ~750 B/750, heapSize ~1.72 KB/1760, currentSize=0 B/0 for f70d4c68e60cf6ec51a282d0cc77a081 in 46ms, sequenceid=12, compaction requested=false; wal=null at 1732631498097 (+12 ms)Cleaning up temporary data from old regions at 1732631498098 (+1 ms)Region opened successfully at 1732631498106 (+8 ms) 2024-11-26T14:31:38,109 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => f70d4c68e60cf6ec51a282d0cc77a081, NAME => 'testDatalossWhenInputError,,1732631497525.f70d4c68e60cf6ec51a282d0cc77a081.', STARTKEY => '', ENDKEY => ''} 2024-11-26T14:31:38,109 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testDatalossWhenInputError,,1732631497525.f70d4c68e60cf6ec51a282d0cc77a081.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-26T14:31:38,109 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for f70d4c68e60cf6ec51a282d0cc77a081 2024-11-26T14:31:38,109 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for f70d4c68e60cf6ec51a282d0cc77a081 2024-11-26T14:31:38,111 INFO [StoreOpener-f70d4c68e60cf6ec51a282d0cc77a081-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region f70d4c68e60cf6ec51a282d0cc77a081 2024-11-26T14:31:38,112 INFO [StoreOpener-f70d4c68e60cf6ec51a282d0cc77a081-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region f70d4c68e60cf6ec51a282d0cc77a081 columnFamilyName a 2024-11-26T14:31:38,112 DEBUG [StoreOpener-f70d4c68e60cf6ec51a282d0cc77a081-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:31:38,117 DEBUG [StoreOpener-f70d4c68e60cf6ec51a282d0cc77a081-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:44999/hbase/data/default/testDatalossWhenInputError/f70d4c68e60cf6ec51a282d0cc77a081/a/8902e370da5b458db77ec6430f459b6a 2024-11-26T14:31:38,117 INFO [StoreOpener-f70d4c68e60cf6ec51a282d0cc77a081-1 {}] regionserver.HStore(327): Store=f70d4c68e60cf6ec51a282d0cc77a081/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-26T14:31:38,117 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for f70d4c68e60cf6ec51a282d0cc77a081 2024-11-26T14:31:38,118 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44999/hbase/data/default/testDatalossWhenInputError/f70d4c68e60cf6ec51a282d0cc77a081 2024-11-26T14:31:38,119 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44999/hbase/data/default/testDatalossWhenInputError/f70d4c68e60cf6ec51a282d0cc77a081 2024-11-26T14:31:38,120 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for f70d4c68e60cf6ec51a282d0cc77a081 2024-11-26T14:31:38,120 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for f70d4c68e60cf6ec51a282d0cc77a081 2024-11-26T14:31:38,122 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for f70d4c68e60cf6ec51a282d0cc77a081 2024-11-26T14:31:38,124 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44999/hbase/data/default/testDatalossWhenInputError/f70d4c68e60cf6ec51a282d0cc77a081/recovered.edits/13.seqid, newMaxSeqId=13, maxSeqId=12 2024-11-26T14:31:38,125 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened f70d4c68e60cf6ec51a282d0cc77a081; next sequenceid=14; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=64387994, jitterRate=-0.04054412245750427}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-26T14:31:38,125 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for f70d4c68e60cf6ec51a282d0cc77a081: Writing region info on filesystem at 1732631498109Initializing all the Stores at 1732631498110 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732631498110Cleaning up temporary data from old regions at 1732631498120 (+10 ms)Region opened successfully at 1732631498125 (+5 ms) 2024-11-26T14:31:38,141 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestAsyncWALReplayCompressed#testDatalossWhenInputError Thread=433 (was 423) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_2043263076_22 at /127.0.0.1:33758 [Receiving block BP-1347678837-172.17.0.3-1732631464290:blk_1073741913_1091] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_2043263076_22 at /127.0.0.1:57846 [Receiving block BP-1347678837-172.17.0.3-1732631464290:blk_1073741913_1091] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-1347678837-172.17.0.3-1732631464290:blk_1073741913_1091, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-24-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-1347678837-172.17.0.3-1732631464290:blk_1073741913_1091, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-24-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_2043263076_22 at /127.0.0.1:47062 [Receiving block BP-1347678837-172.17.0.3-1732631464290:blk_1073741913_1091] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_2043263076_22 at /127.0.0.1:33680 [Waiting for operation #17] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-24-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_2043263076_22 at /127.0.0.1:57778 [Waiting for operation #8] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-1347678837-172.17.0.3-1732631464290:blk_1073741913_1091, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_2043263076_22 at /127.0.0.1:46998 [Waiting for operation #17] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=1265 (was 1183) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=260 (was 239) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=8976 (was 8989) 2024-11-26T14:31:38,142 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1265 is superior to 1024 2024-11-26T14:31:38,153 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestAsyncWALReplayCompressed#testCompactedBulkLoadedFiles Thread=433, OpenFileDescriptor=1265, MaxFileDescriptor=1048576, SystemLoadAverage=260, ProcessCount=11, AvailableMemoryMB=8975 2024-11-26T14:31:38,154 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1265 is superior to 1024 2024-11-26T14:31:38,167 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-26T14:31:38,168 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-26T14:31:38,169 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-26T14:31:38,172 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-36111750, suffix=, logDir=hdfs://localhost:44999/hbase/WALs/hregion-36111750, archiveDir=hdfs://localhost:44999/hbase/oldWALs, maxLogs=32 2024-11-26T14:31:38,185 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-36111750/hregion-36111750.1732631498172, exclude list is [], retry=0 2024-11-26T14:31:38,187 DEBUG [AsyncFSWAL-26-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38147,DS-44b12d44-38d0-4c76-b56f-3ca8d8caa398,DISK] 2024-11-26T14:31:38,188 DEBUG [AsyncFSWAL-26-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36307,DS-dd37de78-b3d5-4d7c-8223-4b15101c21bf,DISK] 2024-11-26T14:31:38,188 DEBUG [AsyncFSWAL-26-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36277,DS-9cb9383d-a6c8-4ba6-83b3-6706331a6539,DISK] 2024-11-26T14:31:38,192 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-36111750/hregion-36111750.1732631498172 2024-11-26T14:31:38,194 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:35081:35081),(127.0.0.1/127.0.0.1:34431:34431),(127.0.0.1/127.0.0.1:40449:40449)] 2024-11-26T14:31:38,194 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => 270dd76d1e735c7a42ab188cb812156e, NAME => 'testCompactedBulkLoadedFiles,,1732631498167.270dd76d1e735c7a42ab188cb812156e.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testCompactedBulkLoadedFiles', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:44999/hbase 2024-11-26T14:31:38,208 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36307 is added to blk_1073741916_1094 (size=63) 2024-11-26T14:31:38,208 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38147 is added to blk_1073741916_1094 (size=63) 2024-11-26T14:31:38,209 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testCompactedBulkLoadedFiles,,1732631498167.270dd76d1e735c7a42ab188cb812156e.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-26T14:31:38,211 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36277 is added to blk_1073741916_1094 (size=63) 2024-11-26T14:31:38,212 INFO [StoreOpener-270dd76d1e735c7a42ab188cb812156e-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 270dd76d1e735c7a42ab188cb812156e 2024-11-26T14:31:38,214 INFO [StoreOpener-270dd76d1e735c7a42ab188cb812156e-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 270dd76d1e735c7a42ab188cb812156e columnFamilyName a 2024-11-26T14:31:38,214 DEBUG [StoreOpener-270dd76d1e735c7a42ab188cb812156e-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:31:38,214 INFO [StoreOpener-270dd76d1e735c7a42ab188cb812156e-1 {}] regionserver.HStore(327): Store=270dd76d1e735c7a42ab188cb812156e/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-26T14:31:38,214 INFO [StoreOpener-270dd76d1e735c7a42ab188cb812156e-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 270dd76d1e735c7a42ab188cb812156e 2024-11-26T14:31:38,215 INFO [StoreOpener-270dd76d1e735c7a42ab188cb812156e-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 270dd76d1e735c7a42ab188cb812156e columnFamilyName b 2024-11-26T14:31:38,215 DEBUG [StoreOpener-270dd76d1e735c7a42ab188cb812156e-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:31:38,216 INFO [StoreOpener-270dd76d1e735c7a42ab188cb812156e-1 {}] regionserver.HStore(327): Store=270dd76d1e735c7a42ab188cb812156e/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-26T14:31:38,216 INFO [StoreOpener-270dd76d1e735c7a42ab188cb812156e-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 270dd76d1e735c7a42ab188cb812156e 2024-11-26T14:31:38,217 INFO [StoreOpener-270dd76d1e735c7a42ab188cb812156e-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 270dd76d1e735c7a42ab188cb812156e columnFamilyName c 2024-11-26T14:31:38,217 DEBUG [StoreOpener-270dd76d1e735c7a42ab188cb812156e-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:31:38,218 INFO [StoreOpener-270dd76d1e735c7a42ab188cb812156e-1 {}] regionserver.HStore(327): Store=270dd76d1e735c7a42ab188cb812156e/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-26T14:31:38,218 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 270dd76d1e735c7a42ab188cb812156e 2024-11-26T14:31:38,219 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44999/hbase/data/default/testCompactedBulkLoadedFiles/270dd76d1e735c7a42ab188cb812156e 2024-11-26T14:31:38,219 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44999/hbase/data/default/testCompactedBulkLoadedFiles/270dd76d1e735c7a42ab188cb812156e 2024-11-26T14:31:38,220 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 270dd76d1e735c7a42ab188cb812156e 2024-11-26T14:31:38,220 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 270dd76d1e735c7a42ab188cb812156e 2024-11-26T14:31:38,221 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testCompactedBulkLoadedFiles descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-26T14:31:38,222 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 270dd76d1e735c7a42ab188cb812156e 2024-11-26T14:31:38,224 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44999/hbase/data/default/testCompactedBulkLoadedFiles/270dd76d1e735c7a42ab188cb812156e/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-26T14:31:38,225 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 270dd76d1e735c7a42ab188cb812156e; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=72667111, jitterRate=0.08282433450222015}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-26T14:31:38,226 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 270dd76d1e735c7a42ab188cb812156e: Writing region info on filesystem at 1732631498209Initializing all the Stores at 1732631498210 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732631498210Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732631498212 (+2 ms)Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732631498212Cleaning up temporary data from old regions at 1732631498220 (+8 ms)Region opened successfully at 1732631498226 (+6 ms) 2024-11-26T14:31:38,226 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 270dd76d1e735c7a42ab188cb812156e, disabling compactions & flushes 2024-11-26T14:31:38,226 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testCompactedBulkLoadedFiles,,1732631498167.270dd76d1e735c7a42ab188cb812156e. 2024-11-26T14:31:38,226 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testCompactedBulkLoadedFiles,,1732631498167.270dd76d1e735c7a42ab188cb812156e. 2024-11-26T14:31:38,226 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testCompactedBulkLoadedFiles,,1732631498167.270dd76d1e735c7a42ab188cb812156e. after waiting 0 ms 2024-11-26T14:31:38,226 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testCompactedBulkLoadedFiles,,1732631498167.270dd76d1e735c7a42ab188cb812156e. 2024-11-26T14:31:38,226 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testCompactedBulkLoadedFiles,,1732631498167.270dd76d1e735c7a42ab188cb812156e. 2024-11-26T14:31:38,227 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 270dd76d1e735c7a42ab188cb812156e: Waiting for close lock at 1732631498226Disabling compacts and flushes for region at 1732631498226Disabling writes for close at 1732631498226Writing region close event to WAL at 1732631498226Closed at 1732631498226 2024-11-26T14:31:38,229 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38147 is added to blk_1073741915_1093 (size=93) 2024-11-26T14:31:38,229 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36307 is added to blk_1073741915_1093 (size=93) 2024-11-26T14:31:38,229 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36277 is added to blk_1073741915_1093 (size=93) 2024-11-26T14:31:38,232 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-11-26T14:31:38,232 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-36111750:(num 1732631498172) 2024-11-26T14:31:38,232 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-26T14:31:38,234 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:44999/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1732631498166, archiveDir=hdfs://localhost:44999/hbase/oldWALs, maxLogs=32 2024-11-26T14:31:38,251 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1732631498166/wal.1732631498235, exclude list is [], retry=0 2024-11-26T14:31:38,253 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36277,DS-9cb9383d-a6c8-4ba6-83b3-6706331a6539,DISK] 2024-11-26T14:31:38,254 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36307,DS-dd37de78-b3d5-4d7c-8223-4b15101c21bf,DISK] 2024-11-26T14:31:38,254 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38147,DS-44b12d44-38d0-4c76-b56f-3ca8d8caa398,DISK] 2024-11-26T14:31:38,258 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1732631498166/wal.1732631498235 2024-11-26T14:31:38,259 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:40449:40449),(127.0.0.1/127.0.0.1:34431:34431),(127.0.0.1/127.0.0.1:35081:35081)] 2024-11-26T14:31:38,259 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 270dd76d1e735c7a42ab188cb812156e, NAME => 'testCompactedBulkLoadedFiles,,1732631498167.270dd76d1e735c7a42ab188cb812156e.', STARTKEY => '', ENDKEY => ''} 2024-11-26T14:31:38,259 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testCompactedBulkLoadedFiles,,1732631498167.270dd76d1e735c7a42ab188cb812156e.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-26T14:31:38,259 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 270dd76d1e735c7a42ab188cb812156e 2024-11-26T14:31:38,259 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 270dd76d1e735c7a42ab188cb812156e 2024-11-26T14:31:38,262 INFO [StoreOpener-270dd76d1e735c7a42ab188cb812156e-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 270dd76d1e735c7a42ab188cb812156e 2024-11-26T14:31:38,263 INFO [StoreOpener-270dd76d1e735c7a42ab188cb812156e-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 270dd76d1e735c7a42ab188cb812156e columnFamilyName a 2024-11-26T14:31:38,263 DEBUG [StoreOpener-270dd76d1e735c7a42ab188cb812156e-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:31:38,264 INFO [StoreOpener-270dd76d1e735c7a42ab188cb812156e-1 {}] regionserver.HStore(327): Store=270dd76d1e735c7a42ab188cb812156e/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-26T14:31:38,264 INFO [StoreOpener-270dd76d1e735c7a42ab188cb812156e-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 270dd76d1e735c7a42ab188cb812156e 2024-11-26T14:31:38,265 INFO [StoreOpener-270dd76d1e735c7a42ab188cb812156e-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 270dd76d1e735c7a42ab188cb812156e columnFamilyName b 2024-11-26T14:31:38,265 DEBUG [StoreOpener-270dd76d1e735c7a42ab188cb812156e-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:31:38,266 INFO [StoreOpener-270dd76d1e735c7a42ab188cb812156e-1 {}] regionserver.HStore(327): Store=270dd76d1e735c7a42ab188cb812156e/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-26T14:31:38,266 INFO [StoreOpener-270dd76d1e735c7a42ab188cb812156e-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 270dd76d1e735c7a42ab188cb812156e 2024-11-26T14:31:38,267 INFO [StoreOpener-270dd76d1e735c7a42ab188cb812156e-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 270dd76d1e735c7a42ab188cb812156e columnFamilyName c 2024-11-26T14:31:38,267 DEBUG [StoreOpener-270dd76d1e735c7a42ab188cb812156e-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:31:38,267 INFO [StoreOpener-270dd76d1e735c7a42ab188cb812156e-1 {}] regionserver.HStore(327): Store=270dd76d1e735c7a42ab188cb812156e/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-26T14:31:38,267 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 270dd76d1e735c7a42ab188cb812156e 2024-11-26T14:31:38,268 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44999/hbase/data/default/testCompactedBulkLoadedFiles/270dd76d1e735c7a42ab188cb812156e 2024-11-26T14:31:38,269 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44999/hbase/data/default/testCompactedBulkLoadedFiles/270dd76d1e735c7a42ab188cb812156e 2024-11-26T14:31:38,270 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 270dd76d1e735c7a42ab188cb812156e 2024-11-26T14:31:38,270 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 270dd76d1e735c7a42ab188cb812156e 2024-11-26T14:31:38,271 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testCompactedBulkLoadedFiles descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-26T14:31:38,272 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 270dd76d1e735c7a42ab188cb812156e 2024-11-26T14:31:38,273 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 270dd76d1e735c7a42ab188cb812156e; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=74749569, jitterRate=0.11385537683963776}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-26T14:31:38,273 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 270dd76d1e735c7a42ab188cb812156e: Writing region info on filesystem at 1732631498260Initializing all the Stores at 1732631498261 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732631498261Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732631498262 (+1 ms)Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732631498262Cleaning up temporary data from old regions at 1732631498270 (+8 ms)Region opened successfully at 1732631498273 (+3 ms) 2024-11-26T14:31:38,333 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44999/hbase/testCompactedBulkLoadedFiles/hfile0 is 32, key is 000/a:a/1732631498333/Put/seqid=0 2024-11-26T14:31:38,340 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36307 is added to blk_1073741918_1096 (size=4875) 2024-11-26T14:31:38,340 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38147 is added to blk_1073741918_1096 (size=4875) 2024-11-26T14:31:38,340 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36277 is added to blk_1073741918_1096 (size=4875) 2024-11-26T14:31:38,344 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44999/hbase/testCompactedBulkLoadedFiles/hfile1 is 32, key is 100/a:a/1732631498343/Put/seqid=0 2024-11-26T14:31:38,350 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38147 is added to blk_1073741919_1097 (size=4875) 2024-11-26T14:31:38,350 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36277 is added to blk_1073741919_1097 (size=4875) 2024-11-26T14:31:38,351 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36307 is added to blk_1073741919_1097 (size=4875) 2024-11-26T14:31:38,353 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44999/hbase/testCompactedBulkLoadedFiles/hfile2 is 32, key is 200/a:a/1732631498353/Put/seqid=0 2024-11-26T14:31:38,362 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36277 is added to blk_1073741920_1098 (size=4875) 2024-11-26T14:31:38,363 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38147 is added to blk_1073741920_1098 (size=4875) 2024-11-26T14:31:38,363 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36307 is added to blk_1073741920_1098 (size=4875) 2024-11-26T14:31:38,363 INFO [Time-limited test {}] regionserver.HStore(614): Validating hfile at hdfs://localhost:44999/hbase/testCompactedBulkLoadedFiles/hfile0 for inclusion in 270dd76d1e735c7a42ab188cb812156e/a 2024-11-26T14:31:38,367 DEBUG [Time-limited test {}] regionserver.HStore(626): HFile bounds: first=000 last=050 2024-11-26T14:31:38,367 DEBUG [Time-limited test {}] regionserver.HStore(628): Region bounds: first= last= 2024-11-26T14:31:38,368 INFO [Time-limited test {}] regionserver.HStore(614): Validating hfile at hdfs://localhost:44999/hbase/testCompactedBulkLoadedFiles/hfile1 for inclusion in 270dd76d1e735c7a42ab188cb812156e/a 2024-11-26T14:31:38,371 DEBUG [Time-limited test {}] regionserver.HStore(626): HFile bounds: first=100 last=150 2024-11-26T14:31:38,371 DEBUG [Time-limited test {}] regionserver.HStore(628): Region bounds: first= last= 2024-11-26T14:31:38,371 INFO [Time-limited test {}] regionserver.HStore(614): Validating hfile at hdfs://localhost:44999/hbase/testCompactedBulkLoadedFiles/hfile2 for inclusion in 270dd76d1e735c7a42ab188cb812156e/a 2024-11-26T14:31:38,375 DEBUG [Time-limited test {}] regionserver.HStore(626): HFile bounds: first=200 last=250 2024-11-26T14:31:38,375 DEBUG [Time-limited test {}] regionserver.HStore(628): Region bounds: first= last= 2024-11-26T14:31:38,375 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 270dd76d1e735c7a42ab188cb812156e 3/3 column families, dataSize=51 B heapSize=896 B 2024-11-26T14:31:38,390 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44999/hbase/data/default/testCompactedBulkLoadedFiles/270dd76d1e735c7a42ab188cb812156e/.tmp/a/c11799e073c04634bdc48cf7d0b165fc is 55, key is testCompactedBulkLoadedFiles/a:a/1732631498274/Put/seqid=0 2024-11-26T14:31:38,398 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36307 is added to blk_1073741921_1099 (size=5107) 2024-11-26T14:31:38,398 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36277 is added to blk_1073741921_1099 (size=5107) 2024-11-26T14:31:38,398 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38147 is added to blk_1073741921_1099 (size=5107) 2024-11-26T14:31:38,401 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51 B at sequenceid=4 (bloomFilter=true), to=hdfs://localhost:44999/hbase/data/default/testCompactedBulkLoadedFiles/270dd76d1e735c7a42ab188cb812156e/.tmp/a/c11799e073c04634bdc48cf7d0b165fc 2024-11-26T14:31:38,407 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44999/hbase/data/default/testCompactedBulkLoadedFiles/270dd76d1e735c7a42ab188cb812156e/.tmp/a/c11799e073c04634bdc48cf7d0b165fc as hdfs://localhost:44999/hbase/data/default/testCompactedBulkLoadedFiles/270dd76d1e735c7a42ab188cb812156e/a/c11799e073c04634bdc48cf7d0b165fc 2024-11-26T14:31:38,413 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44999/hbase/data/default/testCompactedBulkLoadedFiles/270dd76d1e735c7a42ab188cb812156e/a/c11799e073c04634bdc48cf7d0b165fc, entries=1, sequenceid=4, filesize=5.0 K 2024-11-26T14:31:38,414 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~51 B/51, heapSize ~368 B/368, currentSize=0 B/0 for 270dd76d1e735c7a42ab188cb812156e in 39ms, sequenceid=4, compaction requested=false 2024-11-26T14:31:38,414 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for 270dd76d1e735c7a42ab188cb812156e: 2024-11-26T14:31:38,416 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44999/hbase/testCompactedBulkLoadedFiles/hfile0 as hdfs://localhost:44999/hbase/data/default/testCompactedBulkLoadedFiles/270dd76d1e735c7a42ab188cb812156e/a/405dfa9a5a74486481a5f3fd9df4ee8b_SeqId_4_ 2024-11-26T14:31:38,417 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44999/hbase/testCompactedBulkLoadedFiles/hfile1 as hdfs://localhost:44999/hbase/data/default/testCompactedBulkLoadedFiles/270dd76d1e735c7a42ab188cb812156e/a/3248806be56a40daaeaca01493ef97d8_SeqId_4_ 2024-11-26T14:31:38,418 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44999/hbase/testCompactedBulkLoadedFiles/hfile2 as hdfs://localhost:44999/hbase/data/default/testCompactedBulkLoadedFiles/270dd76d1e735c7a42ab188cb812156e/a/bb356cfb33774bc3b6baa6e120344e49_SeqId_4_ 2024-11-26T14:31:38,418 INFO [Time-limited test {}] regionserver.HStore(700): Loaded HFile hdfs://localhost:44999/hbase/testCompactedBulkLoadedFiles/hfile0 into 270dd76d1e735c7a42ab188cb812156e/a as hdfs://localhost:44999/hbase/data/default/testCompactedBulkLoadedFiles/270dd76d1e735c7a42ab188cb812156e/a/405dfa9a5a74486481a5f3fd9df4ee8b_SeqId_4_ - updating store file list. 2024-11-26T14:31:38,422 DEBUG [Time-limited test {}] regionserver.HStoreFile(483): HFile Bloom filter type for 405dfa9a5a74486481a5f3fd9df4ee8b_SeqId_4_: NONE, but ROW specified in column family configuration 2024-11-26T14:31:38,423 INFO [Time-limited test {}] regionserver.HStore(722): Loaded HFile hdfs://localhost:44999/hbase/data/default/testCompactedBulkLoadedFiles/270dd76d1e735c7a42ab188cb812156e/a/405dfa9a5a74486481a5f3fd9df4ee8b_SeqId_4_ into 270dd76d1e735c7a42ab188cb812156e/a 2024-11-26T14:31:38,423 INFO [Time-limited test {}] regionserver.HStore(706): Successfully loaded hdfs://localhost:44999/hbase/testCompactedBulkLoadedFiles/hfile0 into 270dd76d1e735c7a42ab188cb812156e/a (new location: hdfs://localhost:44999/hbase/data/default/testCompactedBulkLoadedFiles/270dd76d1e735c7a42ab188cb812156e/a/405dfa9a5a74486481a5f3fd9df4ee8b_SeqId_4_) 2024-11-26T14:31:38,424 INFO [Time-limited test {}] regionserver.HStore(700): Loaded HFile hdfs://localhost:44999/hbase/testCompactedBulkLoadedFiles/hfile1 into 270dd76d1e735c7a42ab188cb812156e/a as hdfs://localhost:44999/hbase/data/default/testCompactedBulkLoadedFiles/270dd76d1e735c7a42ab188cb812156e/a/3248806be56a40daaeaca01493ef97d8_SeqId_4_ - updating store file list. 2024-11-26T14:31:38,428 DEBUG [Time-limited test {}] regionserver.HStoreFile(483): HFile Bloom filter type for 3248806be56a40daaeaca01493ef97d8_SeqId_4_: NONE, but ROW specified in column family configuration 2024-11-26T14:31:38,428 INFO [Time-limited test {}] regionserver.HStore(722): Loaded HFile hdfs://localhost:44999/hbase/data/default/testCompactedBulkLoadedFiles/270dd76d1e735c7a42ab188cb812156e/a/3248806be56a40daaeaca01493ef97d8_SeqId_4_ into 270dd76d1e735c7a42ab188cb812156e/a 2024-11-26T14:31:38,428 INFO [Time-limited test {}] regionserver.HStore(706): Successfully loaded hdfs://localhost:44999/hbase/testCompactedBulkLoadedFiles/hfile1 into 270dd76d1e735c7a42ab188cb812156e/a (new location: hdfs://localhost:44999/hbase/data/default/testCompactedBulkLoadedFiles/270dd76d1e735c7a42ab188cb812156e/a/3248806be56a40daaeaca01493ef97d8_SeqId_4_) 2024-11-26T14:31:38,429 INFO [Time-limited test {}] regionserver.HStore(700): Loaded HFile hdfs://localhost:44999/hbase/testCompactedBulkLoadedFiles/hfile2 into 270dd76d1e735c7a42ab188cb812156e/a as hdfs://localhost:44999/hbase/data/default/testCompactedBulkLoadedFiles/270dd76d1e735c7a42ab188cb812156e/a/bb356cfb33774bc3b6baa6e120344e49_SeqId_4_ - updating store file list. 2024-11-26T14:31:38,434 DEBUG [Time-limited test {}] regionserver.HStoreFile(483): HFile Bloom filter type for bb356cfb33774bc3b6baa6e120344e49_SeqId_4_: NONE, but ROW specified in column family configuration 2024-11-26T14:31:38,434 INFO [Time-limited test {}] regionserver.HStore(722): Loaded HFile hdfs://localhost:44999/hbase/data/default/testCompactedBulkLoadedFiles/270dd76d1e735c7a42ab188cb812156e/a/bb356cfb33774bc3b6baa6e120344e49_SeqId_4_ into 270dd76d1e735c7a42ab188cb812156e/a 2024-11-26T14:31:38,434 INFO [Time-limited test {}] regionserver.HStore(706): Successfully loaded hdfs://localhost:44999/hbase/testCompactedBulkLoadedFiles/hfile2 into 270dd76d1e735c7a42ab188cb812156e/a (new location: hdfs://localhost:44999/hbase/data/default/testCompactedBulkLoadedFiles/270dd76d1e735c7a42ab188cb812156e/a/bb356cfb33774bc3b6baa6e120344e49_SeqId_4_) 2024-11-26T14:31:38,443 DEBUG [Time-limited test {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-26T14:31:38,443 DEBUG [Time-limited test {}] regionserver.HStore(1541): 270dd76d1e735c7a42ab188cb812156e/a is initiating major compaction (all files) 2024-11-26T14:31:38,443 INFO [Time-limited test {}] regionserver.HRegion(2416): Starting compaction of 270dd76d1e735c7a42ab188cb812156e/a in testCompactedBulkLoadedFiles,,1732631498167.270dd76d1e735c7a42ab188cb812156e. 2024-11-26T14:31:38,443 INFO [Time-limited test {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44999/hbase/data/default/testCompactedBulkLoadedFiles/270dd76d1e735c7a42ab188cb812156e/a/c11799e073c04634bdc48cf7d0b165fc, hdfs://localhost:44999/hbase/data/default/testCompactedBulkLoadedFiles/270dd76d1e735c7a42ab188cb812156e/a/405dfa9a5a74486481a5f3fd9df4ee8b_SeqId_4_, hdfs://localhost:44999/hbase/data/default/testCompactedBulkLoadedFiles/270dd76d1e735c7a42ab188cb812156e/a/3248806be56a40daaeaca01493ef97d8_SeqId_4_, hdfs://localhost:44999/hbase/data/default/testCompactedBulkLoadedFiles/270dd76d1e735c7a42ab188cb812156e/a/bb356cfb33774bc3b6baa6e120344e49_SeqId_4_] into tmpdir=hdfs://localhost:44999/hbase/data/default/testCompactedBulkLoadedFiles/270dd76d1e735c7a42ab188cb812156e/.tmp, totalSize=19.3 K 2024-11-26T14:31:38,444 DEBUG [Time-limited test {}] compactions.Compactor(225): Compacting c11799e073c04634bdc48cf7d0b165fc, keycount=1, bloomtype=ROW, size=5.0 K, encoding=NONE, compression=NONE, seqNum=4, earliestPutTs=1732631498274 2024-11-26T14:31:38,445 DEBUG [Time-limited test {}] compactions.Compactor(225): Compacting 405dfa9a5a74486481a5f3fd9df4ee8b_SeqId_4_, keycount=10, bloomtype=NONE, size=4.8 K, encoding=NONE, compression=NONE, seqNum=4, earliestPutTs=-9223372036854775808 2024-11-26T14:31:38,445 DEBUG [Time-limited test {}] compactions.Compactor(225): Compacting 3248806be56a40daaeaca01493ef97d8_SeqId_4_, keycount=10, bloomtype=NONE, size=4.8 K, encoding=NONE, compression=NONE, seqNum=4, earliestPutTs=-9223372036854775808 2024-11-26T14:31:38,445 DEBUG [Time-limited test {}] compactions.Compactor(225): Compacting bb356cfb33774bc3b6baa6e120344e49_SeqId_4_, keycount=10, bloomtype=NONE, size=4.8 K, encoding=NONE, compression=NONE, seqNum=4, earliestPutTs=-9223372036854775808 2024-11-26T14:31:38,458 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44999/hbase/data/default/testCompactedBulkLoadedFiles/270dd76d1e735c7a42ab188cb812156e/.tmp/a/a978e036cf9540d195544ad786b58758 is 55, key is testCompactedBulkLoadedFiles/a:a/1732631498274/Put/seqid=0 2024-11-26T14:31:38,466 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36277 is added to blk_1073741922_1100 (size=6154) 2024-11-26T14:31:38,466 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36307 is added to blk_1073741922_1100 (size=6154) 2024-11-26T14:31:38,466 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38147 is added to blk_1073741922_1100 (size=6154) 2024-11-26T14:31:38,473 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44999/hbase/data/default/testCompactedBulkLoadedFiles/270dd76d1e735c7a42ab188cb812156e/.tmp/a/a978e036cf9540d195544ad786b58758 as hdfs://localhost:44999/hbase/data/default/testCompactedBulkLoadedFiles/270dd76d1e735c7a42ab188cb812156e/a/a978e036cf9540d195544ad786b58758 2024-11-26T14:31:38,480 INFO [Time-limited test {}] regionserver.HStore(1337): Completed major compaction of 4 (all) file(s) in 270dd76d1e735c7a42ab188cb812156e/a of 270dd76d1e735c7a42ab188cb812156e into a978e036cf9540d195544ad786b58758(size=6.0 K), total size for store is 6.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T14:31:38,480 DEBUG [Time-limited test {}] regionserver.HRegion(2446): Compaction status journal for 270dd76d1e735c7a42ab188cb812156e: 2024-11-26T14:31:38,480 DEBUG [Time-limited test {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 0 store files, 0 compacting, 0 eligible, 16 blocking 2024-11-26T14:31:38,480 DEBUG [Time-limited test {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 0 store files, 0 compacting, 0 eligible, 16 blocking 2024-11-26T14:31:38,515 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:44999/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1732631498166/wal.1732631498235, size=0 (0bytes) 2024-11-26T14:31:38,515 WARN [Time-limited test {}] wal.WALSplitter(453): File hdfs://localhost:44999/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1732631498166/wal.1732631498235 might be still open, length is 0 2024-11-26T14:31:38,515 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:44999/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1732631498166/wal.1732631498235 2024-11-26T14:31:38,516 WARN [IPC Server handler 4 on default port 44999 {}] namenode.FSNamesystem(3854): DIR* NameSystem.internalReleaseLease: File /hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1732631498166/wal.1732631498235 has not been closed. Lease recovery is in progress. RecoveryId = 1101 for block blk_1073741917_1095 2024-11-26T14:31:38,516 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=0 on file=hdfs://localhost:44999/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1732631498166/wal.1732631498235 after 1ms 2024-11-26T14:31:38,994 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testDatalossWhenInputError 2024-11-26T14:31:38,994 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testDatalossWhenInputError Metrics about Tables on a single HBase RegionServer 2024-11-26T14:31:38,996 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testCompactedBulkLoadedFiles 2024-11-26T14:31:38,996 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testCompactedBulkLoadedFiles Metrics about Tables on a single HBase RegionServer 2024-11-26T14:31:40,854 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_2043263076_22 at /127.0.0.1:57878 [Receiving block BP-1347678837-172.17.0.3-1732631464290:blk_1073741917_1095] {}] datanode.DataXceiver(331): 127.0.0.1:36277:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:57878 dst: /127.0.0.1:36277 java.io.InterruptedIOException: Interrupted while waiting for IO on channel java.nio.channels.SocketChannel[connected local=localhost/127.0.0.1:36277 remote=/127.0.0.1:57878]. Total timeout mills is 60000, 57627 millis timeout left. at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:350) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?] at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?] at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?] at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?] at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?] at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T14:31:40,855 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_2043263076_22 at /127.0.0.1:47106 [Receiving block BP-1347678837-172.17.0.3-1732631464290:blk_1073741917_1095] {}] datanode.DataXceiver(331): 127.0.0.1:36307:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:47106 dst: /127.0.0.1:36307 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T14:31:40,855 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_2043263076_22 at /127.0.0.1:33798 [Receiving block BP-1347678837-172.17.0.3-1732631464290:blk_1073741917_1095] {}] datanode.DataXceiver(331): 127.0.0.1:38147:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:33798 dst: /127.0.0.1:38147 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T14:31:40,863 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36277 is added to blk_1073741917_1101 (size=1368) 2024-11-26T14:31:40,863 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36307 is added to blk_1073741917_1101 (size=1368) 2024-11-26T14:31:40,864 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38147 is added to blk_1073741917_1101 (size=1368) 2024-11-26T14:31:42,518 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=1 on file=hdfs://localhost:44999/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1732631498166/wal.1732631498235 after 4002ms 2024-11-26T14:31:42,524 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:44999/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1732631498166/wal.1732631498235: isRecoveredEdits=false, hasTagCompression=true, hasValueCompression=false, valueCompressionType=null 2024-11-26T14:31:42,524 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:44999/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1732631498166/wal.1732631498235 took 4009ms 2024-11-26T14:31:42,527 INFO [Time-limited test {}] wal.WALSplitter(493): EOF from hdfs://localhost:44999/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1732631498166/wal.1732631498235; continuing. 2024-11-26T14:31:42,527 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:44999/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1732631498166/wal.1732631498235 so closing down 2024-11-26T14:31:42,527 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-26T14:31:42,530 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000003-wal.1732631498235.temp 2024-11-26T14:31:42,532 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:44999/hbase/data/default/testCompactedBulkLoadedFiles/270dd76d1e735c7a42ab188cb812156e/recovered.edits/0000000000000000003-wal.1732631498235.temp 2024-11-26T14:31:42,532 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-11-26T14:31:42,540 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36307 is added to blk_1073741923_1102 (size=635) 2024-11-26T14:31:42,540 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36277 is added to blk_1073741923_1102 (size=635) 2024-11-26T14:31:42,541 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38147 is added to blk_1073741923_1102 (size=635) 2024-11-26T14:31:42,541 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:44999/hbase/data/default/testCompactedBulkLoadedFiles/270dd76d1e735c7a42ab188cb812156e/recovered.edits/0000000000000000003-wal.1732631498235.temp (wrote 2 edits, skipped 0 edits in 0 ms) 2024-11-26T14:31:42,542 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:44999/hbase/data/default/testCompactedBulkLoadedFiles/270dd76d1e735c7a42ab188cb812156e/recovered.edits/0000000000000000003-wal.1732631498235.temp to hdfs://localhost:44999/hbase/data/default/testCompactedBulkLoadedFiles/270dd76d1e735c7a42ab188cb812156e/recovered.edits/0000000000000000008 2024-11-26T14:31:42,542 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 5 edits across 1 Regions in 17 ms; skipped=3; WAL=hdfs://localhost:44999/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1732631498166/wal.1732631498235, size=0, length=0, corrupted=false, cancelled=false 2024-11-26T14:31:42,543 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:44999/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1732631498166/wal.1732631498235, journal: Splitting hdfs://localhost:44999/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1732631498166/wal.1732631498235, size=0 (0bytes) at 1732631498515Finishing writing output for hdfs://localhost:44999/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1732631498166/wal.1732631498235 so closing down at 1732631502527 (+4012 ms)Creating recovered edits writer path=hdfs://localhost:44999/hbase/data/default/testCompactedBulkLoadedFiles/270dd76d1e735c7a42ab188cb812156e/recovered.edits/0000000000000000003-wal.1732631498235.temp at 1732631502532 (+5 ms)3 split writer threads finished at 1732631502532Closed recovered edits writer path=hdfs://localhost:44999/hbase/data/default/testCompactedBulkLoadedFiles/270dd76d1e735c7a42ab188cb812156e/recovered.edits/0000000000000000003-wal.1732631498235.temp (wrote 2 edits, skipped 0 edits in 0 ms) at 1732631502541 (+9 ms)Rename recovered edits hdfs://localhost:44999/hbase/data/default/testCompactedBulkLoadedFiles/270dd76d1e735c7a42ab188cb812156e/recovered.edits/0000000000000000003-wal.1732631498235.temp to hdfs://localhost:44999/hbase/data/default/testCompactedBulkLoadedFiles/270dd76d1e735c7a42ab188cb812156e/recovered.edits/0000000000000000008 at 1732631502542 (+1 ms)Processed 5 edits across 1 Regions in 17 ms; skipped=3; WAL=hdfs://localhost:44999/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1732631498166/wal.1732631498235, size=0, length=0, corrupted=false, cancelled=false at 1732631502542 2024-11-26T14:31:42,544 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:44999/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1732631498166/wal.1732631498235 to hdfs://localhost:44999/hbase/oldWALs/wal.1732631498235 2024-11-26T14:31:42,545 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:44999/hbase/data/default/testCompactedBulkLoadedFiles/270dd76d1e735c7a42ab188cb812156e/recovered.edits/0000000000000000008 2024-11-26T14:31:42,545 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-26T14:31:42,546 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:44999/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1732631498166, archiveDir=hdfs://localhost:44999/hbase/oldWALs, maxLogs=32 2024-11-26T14:31:42,560 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1732631498166/wal.1732631502547, exclude list is [], retry=0 2024-11-26T14:31:42,562 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36277,DS-9cb9383d-a6c8-4ba6-83b3-6706331a6539,DISK] 2024-11-26T14:31:42,562 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36307,DS-dd37de78-b3d5-4d7c-8223-4b15101c21bf,DISK] 2024-11-26T14:31:42,563 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38147,DS-44b12d44-38d0-4c76-b56f-3ca8d8caa398,DISK] 2024-11-26T14:31:42,564 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1732631498166/wal.1732631502547 2024-11-26T14:31:42,564 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:40449:40449),(127.0.0.1/127.0.0.1:34431:34431),(127.0.0.1/127.0.0.1:35081:35081)] 2024-11-26T14:31:42,564 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 270dd76d1e735c7a42ab188cb812156e, NAME => 'testCompactedBulkLoadedFiles,,1732631498167.270dd76d1e735c7a42ab188cb812156e.', STARTKEY => '', ENDKEY => ''} 2024-11-26T14:31:42,565 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testCompactedBulkLoadedFiles,,1732631498167.270dd76d1e735c7a42ab188cb812156e.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-26T14:31:42,565 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 270dd76d1e735c7a42ab188cb812156e 2024-11-26T14:31:42,565 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 270dd76d1e735c7a42ab188cb812156e 2024-11-26T14:31:42,566 INFO [StoreOpener-270dd76d1e735c7a42ab188cb812156e-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 270dd76d1e735c7a42ab188cb812156e 2024-11-26T14:31:42,567 INFO [StoreOpener-270dd76d1e735c7a42ab188cb812156e-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 270dd76d1e735c7a42ab188cb812156e columnFamilyName a 2024-11-26T14:31:42,567 DEBUG [StoreOpener-270dd76d1e735c7a42ab188cb812156e-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:31:42,573 DEBUG [StoreFileOpener-270dd76d1e735c7a42ab188cb812156e-a-1 {}] regionserver.HStoreFile(483): HFile Bloom filter type for 3248806be56a40daaeaca01493ef97d8_SeqId_4_: NONE, but ROW specified in column family configuration 2024-11-26T14:31:42,573 DEBUG [StoreOpener-270dd76d1e735c7a42ab188cb812156e-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:44999/hbase/data/default/testCompactedBulkLoadedFiles/270dd76d1e735c7a42ab188cb812156e/a/3248806be56a40daaeaca01493ef97d8_SeqId_4_ 2024-11-26T14:31:42,577 DEBUG [StoreFileOpener-270dd76d1e735c7a42ab188cb812156e-a-1 {}] regionserver.HStoreFile(483): HFile Bloom filter type for 405dfa9a5a74486481a5f3fd9df4ee8b_SeqId_4_: NONE, but ROW specified in column family configuration 2024-11-26T14:31:42,577 DEBUG [StoreOpener-270dd76d1e735c7a42ab188cb812156e-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:44999/hbase/data/default/testCompactedBulkLoadedFiles/270dd76d1e735c7a42ab188cb812156e/a/405dfa9a5a74486481a5f3fd9df4ee8b_SeqId_4_ 2024-11-26T14:31:42,580 DEBUG [StoreOpener-270dd76d1e735c7a42ab188cb812156e-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:44999/hbase/data/default/testCompactedBulkLoadedFiles/270dd76d1e735c7a42ab188cb812156e/a/a978e036cf9540d195544ad786b58758 2024-11-26T14:31:42,583 DEBUG [StoreFileOpener-270dd76d1e735c7a42ab188cb812156e-a-1 {}] regionserver.HStoreFile(483): HFile Bloom filter type for bb356cfb33774bc3b6baa6e120344e49_SeqId_4_: NONE, but ROW specified in column family configuration 2024-11-26T14:31:42,583 DEBUG [StoreOpener-270dd76d1e735c7a42ab188cb812156e-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:44999/hbase/data/default/testCompactedBulkLoadedFiles/270dd76d1e735c7a42ab188cb812156e/a/bb356cfb33774bc3b6baa6e120344e49_SeqId_4_ 2024-11-26T14:31:42,587 DEBUG [StoreOpener-270dd76d1e735c7a42ab188cb812156e-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:44999/hbase/data/default/testCompactedBulkLoadedFiles/270dd76d1e735c7a42ab188cb812156e/a/c11799e073c04634bdc48cf7d0b165fc 2024-11-26T14:31:42,587 WARN [StoreOpener-270dd76d1e735c7a42ab188cb812156e-1 {}] regionserver.StoreEngine(317): Clearing the compacted storefile hdfs://localhost:44999/hbase/data/default/testCompactedBulkLoadedFiles/270dd76d1e735c7a42ab188cb812156e/a/3248806be56a40daaeaca01493ef97d8_SeqId_4_ from org.apache.hadoop.hbase.regionserver.DefaultStoreEngine@67d729ce 2024-11-26T14:31:42,587 WARN [StoreOpener-270dd76d1e735c7a42ab188cb812156e-1 {}] regionserver.StoreEngine(317): Clearing the compacted storefile hdfs://localhost:44999/hbase/data/default/testCompactedBulkLoadedFiles/270dd76d1e735c7a42ab188cb812156e/a/405dfa9a5a74486481a5f3fd9df4ee8b_SeqId_4_ from org.apache.hadoop.hbase.regionserver.DefaultStoreEngine@67d729ce 2024-11-26T14:31:42,587 WARN [StoreOpener-270dd76d1e735c7a42ab188cb812156e-1 {}] regionserver.StoreEngine(317): Clearing the compacted storefile hdfs://localhost:44999/hbase/data/default/testCompactedBulkLoadedFiles/270dd76d1e735c7a42ab188cb812156e/a/bb356cfb33774bc3b6baa6e120344e49_SeqId_4_ from org.apache.hadoop.hbase.regionserver.DefaultStoreEngine@67d729ce 2024-11-26T14:31:42,587 WARN [StoreOpener-270dd76d1e735c7a42ab188cb812156e-1 {}] regionserver.StoreEngine(317): Clearing the compacted storefile hdfs://localhost:44999/hbase/data/default/testCompactedBulkLoadedFiles/270dd76d1e735c7a42ab188cb812156e/a/c11799e073c04634bdc48cf7d0b165fc from org.apache.hadoop.hbase.regionserver.DefaultStoreEngine@67d729ce 2024-11-26T14:31:42,587 DEBUG [StoreOpener-270dd76d1e735c7a42ab188cb812156e-1 {}] regionserver.StoreEngine(327): Moving the files [hdfs://localhost:44999/hbase/data/default/testCompactedBulkLoadedFiles/270dd76d1e735c7a42ab188cb812156e/a/3248806be56a40daaeaca01493ef97d8_SeqId_4_, hdfs://localhost:44999/hbase/data/default/testCompactedBulkLoadedFiles/270dd76d1e735c7a42ab188cb812156e/a/405dfa9a5a74486481a5f3fd9df4ee8b_SeqId_4_, hdfs://localhost:44999/hbase/data/default/testCompactedBulkLoadedFiles/270dd76d1e735c7a42ab188cb812156e/a/bb356cfb33774bc3b6baa6e120344e49_SeqId_4_, hdfs://localhost:44999/hbase/data/default/testCompactedBulkLoadedFiles/270dd76d1e735c7a42ab188cb812156e/a/c11799e073c04634bdc48cf7d0b165fc] to archive 2024-11-26T14:31:42,588 DEBUG [StoreOpener-270dd76d1e735c7a42ab188cb812156e-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-26T14:31:42,590 DEBUG [StoreOpener-270dd76d1e735c7a42ab188cb812156e-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44999/hbase/data/default/testCompactedBulkLoadedFiles/270dd76d1e735c7a42ab188cb812156e/a/3248806be56a40daaeaca01493ef97d8_SeqId_4_ to hdfs://localhost:44999/hbase/archive/data/default/testCompactedBulkLoadedFiles/270dd76d1e735c7a42ab188cb812156e/a/3248806be56a40daaeaca01493ef97d8_SeqId_4_ 2024-11-26T14:31:42,591 DEBUG [StoreOpener-270dd76d1e735c7a42ab188cb812156e-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44999/hbase/data/default/testCompactedBulkLoadedFiles/270dd76d1e735c7a42ab188cb812156e/a/405dfa9a5a74486481a5f3fd9df4ee8b_SeqId_4_ to hdfs://localhost:44999/hbase/archive/data/default/testCompactedBulkLoadedFiles/270dd76d1e735c7a42ab188cb812156e/a/405dfa9a5a74486481a5f3fd9df4ee8b_SeqId_4_ 2024-11-26T14:31:42,592 DEBUG [StoreOpener-270dd76d1e735c7a42ab188cb812156e-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44999/hbase/data/default/testCompactedBulkLoadedFiles/270dd76d1e735c7a42ab188cb812156e/a/bb356cfb33774bc3b6baa6e120344e49_SeqId_4_ to hdfs://localhost:44999/hbase/archive/data/default/testCompactedBulkLoadedFiles/270dd76d1e735c7a42ab188cb812156e/a/bb356cfb33774bc3b6baa6e120344e49_SeqId_4_ 2024-11-26T14:31:42,593 DEBUG [StoreOpener-270dd76d1e735c7a42ab188cb812156e-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44999/hbase/data/default/testCompactedBulkLoadedFiles/270dd76d1e735c7a42ab188cb812156e/a/c11799e073c04634bdc48cf7d0b165fc to hdfs://localhost:44999/hbase/archive/data/default/testCompactedBulkLoadedFiles/270dd76d1e735c7a42ab188cb812156e/a/c11799e073c04634bdc48cf7d0b165fc 2024-11-26T14:31:42,593 INFO [StoreOpener-270dd76d1e735c7a42ab188cb812156e-1 {}] regionserver.HStore(327): Store=270dd76d1e735c7a42ab188cb812156e/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-26T14:31:42,593 INFO [StoreOpener-270dd76d1e735c7a42ab188cb812156e-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 270dd76d1e735c7a42ab188cb812156e 2024-11-26T14:31:42,594 INFO [StoreOpener-270dd76d1e735c7a42ab188cb812156e-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 270dd76d1e735c7a42ab188cb812156e columnFamilyName b 2024-11-26T14:31:42,594 DEBUG [StoreOpener-270dd76d1e735c7a42ab188cb812156e-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:31:42,594 INFO [StoreOpener-270dd76d1e735c7a42ab188cb812156e-1 {}] regionserver.HStore(327): Store=270dd76d1e735c7a42ab188cb812156e/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-26T14:31:42,594 INFO [StoreOpener-270dd76d1e735c7a42ab188cb812156e-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 270dd76d1e735c7a42ab188cb812156e 2024-11-26T14:31:42,595 INFO [StoreOpener-270dd76d1e735c7a42ab188cb812156e-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 270dd76d1e735c7a42ab188cb812156e columnFamilyName c 2024-11-26T14:31:42,595 DEBUG [StoreOpener-270dd76d1e735c7a42ab188cb812156e-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:31:42,595 INFO [StoreOpener-270dd76d1e735c7a42ab188cb812156e-1 {}] regionserver.HStore(327): Store=270dd76d1e735c7a42ab188cb812156e/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-26T14:31:42,596 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 270dd76d1e735c7a42ab188cb812156e 2024-11-26T14:31:42,597 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44999/hbase/data/default/testCompactedBulkLoadedFiles/270dd76d1e735c7a42ab188cb812156e 2024-11-26T14:31:42,598 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:44999/hbase/data/default/testCompactedBulkLoadedFiles/270dd76d1e735c7a42ab188cb812156e 2024-11-26T14:31:42,599 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:44999/hbase/data/default/testCompactedBulkLoadedFiles/270dd76d1e735c7a42ab188cb812156e/recovered.edits/0000000000000000008 2024-11-26T14:31:42,600 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:44999/hbase/data/default/testCompactedBulkLoadedFiles/270dd76d1e735c7a42ab188cb812156e/recovered.edits/0000000000000000008: isRecoveredEdits=true, hasTagCompression=true, hasValueCompression=false, valueCompressionType=null 2024-11-26T14:31:42,602 DEBUG [Time-limited test {}] regionserver.HRegion(5836): 270dd76d1e735c7a42ab188cb812156e : Replaying compaction marker table_name: "testCompactedBulkLoadedFiles" encoded_region_name: "270dd76d1e735c7a42ab188cb812156e" family_name: "a" compaction_input: "c11799e073c04634bdc48cf7d0b165fc" compaction_input: "405dfa9a5a74486481a5f3fd9df4ee8b_SeqId_4_" compaction_input: "3248806be56a40daaeaca01493ef97d8_SeqId_4_" compaction_input: "bb356cfb33774bc3b6baa6e120344e49_SeqId_4_" compaction_output: "a978e036cf9540d195544ad786b58758" store_home_dir: "a" region_name: "testCompactedBulkLoadedFiles,,1732631498167.270dd76d1e735c7a42ab188cb812156e." with seqId=9223372036854775807 and lastReplayedOpenRegionSeqId=-1 2024-11-26T14:31:42,602 DEBUG [Time-limited test {}] regionserver.HStore(1354): Completing compaction from the WAL marker 2024-11-26T14:31:42,602 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 0, skipped 2, firstSequenceIdInLog=3, maxSequenceIdInLog=8, path=hdfs://localhost:44999/hbase/data/default/testCompactedBulkLoadedFiles/270dd76d1e735c7a42ab188cb812156e/recovered.edits/0000000000000000008 2024-11-26T14:31:42,603 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:44999/hbase/data/default/testCompactedBulkLoadedFiles/270dd76d1e735c7a42ab188cb812156e/recovered.edits/0000000000000000008 2024-11-26T14:31:42,604 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 270dd76d1e735c7a42ab188cb812156e 2024-11-26T14:31:42,604 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 270dd76d1e735c7a42ab188cb812156e 2024-11-26T14:31:42,604 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testCompactedBulkLoadedFiles descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-26T14:31:42,605 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 270dd76d1e735c7a42ab188cb812156e 2024-11-26T14:31:42,607 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44999/hbase/data/default/testCompactedBulkLoadedFiles/270dd76d1e735c7a42ab188cb812156e/recovered.edits/8.seqid, newMaxSeqId=8, maxSeqId=1 2024-11-26T14:31:42,608 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 270dd76d1e735c7a42ab188cb812156e; next sequenceid=9; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=68906625, jitterRate=0.026788726449012756}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-26T14:31:42,608 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 270dd76d1e735c7a42ab188cb812156e: Writing region info on filesystem at 1732631502565Initializing all the Stores at 1732631502566 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732631502566Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732631502566Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732631502566Cleaning up temporary data from old regions at 1732631502604 (+38 ms)Region opened successfully at 1732631502608 (+4 ms) 2024-11-26T14:31:42,610 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 270dd76d1e735c7a42ab188cb812156e, disabling compactions & flushes 2024-11-26T14:31:42,610 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testCompactedBulkLoadedFiles,,1732631498167.270dd76d1e735c7a42ab188cb812156e. 2024-11-26T14:31:42,610 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testCompactedBulkLoadedFiles,,1732631498167.270dd76d1e735c7a42ab188cb812156e. 2024-11-26T14:31:42,610 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testCompactedBulkLoadedFiles,,1732631498167.270dd76d1e735c7a42ab188cb812156e. after waiting 0 ms 2024-11-26T14:31:42,611 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testCompactedBulkLoadedFiles,,1732631498167.270dd76d1e735c7a42ab188cb812156e. 2024-11-26T14:31:42,611 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testCompactedBulkLoadedFiles,,1732631498167.270dd76d1e735c7a42ab188cb812156e. 2024-11-26T14:31:42,611 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 270dd76d1e735c7a42ab188cb812156e: Waiting for close lock at 1732631502610Disabling compacts and flushes for region at 1732631502610Disabling writes for close at 1732631502611 (+1 ms)Writing region close event to WAL at 1732631502611Closed at 1732631502611 2024-11-26T14:31:42,614 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38147 is added to blk_1073741924_1103 (size=93) 2024-11-26T14:31:42,614 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36277 is added to blk_1073741924_1103 (size=93) 2024-11-26T14:31:42,614 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36307 is added to blk_1073741924_1103 (size=93) 2024-11-26T14:31:42,616 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-11-26T14:31:42,616 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL wal:(num 1732631502547) 2024-11-26T14:31:42,628 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestAsyncWALReplayCompressed#testCompactedBulkLoadedFiles Thread=446 (was 433) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1446611275_22 at /127.0.0.1:42982 [Waiting for operation #5] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-26-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1446611275_22 at /127.0.0.1:38938 [Waiting for operation #7] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: LeaseRenewer:jenkinstestCompactedBulkLoadedFiles@localhost:44999 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Client (1987708253) connection to localhost/127.0.0.1:44999 from jenkinstestCompactedBulkLoadedFiles java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1446611275_22 at /127.0.0.1:47404 [Waiting for operation #7] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-26-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-26-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=1348 (was 1265) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=260 (was 260), ProcessCount=11 (was 11), AvailableMemoryMB=8974 (was 8975) 2024-11-26T14:31:42,628 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1348 is superior to 1024 2024-11-26T14:31:42,640 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestAsyncWALReplayCompressed#testReplayEditsWrittenViaHRegion Thread=446, OpenFileDescriptor=1348, MaxFileDescriptor=1048576, SystemLoadAverage=260, ProcessCount=11, AvailableMemoryMB=8974 2024-11-26T14:31:42,641 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1348 is superior to 1024 2024-11-26T14:31:42,656 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-26T14:31:42,658 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-26T14:31:42,658 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-26T14:31:42,661 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-71861647, suffix=, logDir=hdfs://localhost:44999/hbase/WALs/hregion-71861647, archiveDir=hdfs://localhost:44999/hbase/oldWALs, maxLogs=32 2024-11-26T14:31:42,671 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-71861647/hregion-71861647.1732631502661, exclude list is [], retry=0 2024-11-26T14:31:42,674 DEBUG [AsyncFSWAL-28-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38147,DS-44b12d44-38d0-4c76-b56f-3ca8d8caa398,DISK] 2024-11-26T14:31:42,674 DEBUG [AsyncFSWAL-28-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36307,DS-dd37de78-b3d5-4d7c-8223-4b15101c21bf,DISK] 2024-11-26T14:31:42,675 DEBUG [AsyncFSWAL-28-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36277,DS-9cb9383d-a6c8-4ba6-83b3-6706331a6539,DISK] 2024-11-26T14:31:42,676 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-71861647/hregion-71861647.1732631502661 2024-11-26T14:31:42,677 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:35081:35081),(127.0.0.1/127.0.0.1:34431:34431),(127.0.0.1/127.0.0.1:40449:40449)] 2024-11-26T14:31:42,677 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => 1c802a034a40c9c3ec031e1f2de84d81, NAME => 'testReplayEditsWrittenViaHRegion,,1732631502657.1c802a034a40c9c3ec031e1f2de84d81.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testReplayEditsWrittenViaHRegion', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:44999/hbase 2024-11-26T14:31:42,684 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36277 is added to blk_1073741926_1105 (size=67) 2024-11-26T14:31:42,684 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36307 is added to blk_1073741926_1105 (size=67) 2024-11-26T14:31:42,684 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38147 is added to blk_1073741926_1105 (size=67) 2024-11-26T14:31:42,685 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenViaHRegion,,1732631502657.1c802a034a40c9c3ec031e1f2de84d81.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-26T14:31:42,686 INFO [StoreOpener-1c802a034a40c9c3ec031e1f2de84d81-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 1c802a034a40c9c3ec031e1f2de84d81 2024-11-26T14:31:42,688 INFO [StoreOpener-1c802a034a40c9c3ec031e1f2de84d81-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1c802a034a40c9c3ec031e1f2de84d81 columnFamilyName a 2024-11-26T14:31:42,688 DEBUG [StoreOpener-1c802a034a40c9c3ec031e1f2de84d81-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:31:42,688 INFO [StoreOpener-1c802a034a40c9c3ec031e1f2de84d81-1 {}] regionserver.HStore(327): Store=1c802a034a40c9c3ec031e1f2de84d81/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-26T14:31:42,689 INFO [StoreOpener-1c802a034a40c9c3ec031e1f2de84d81-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 1c802a034a40c9c3ec031e1f2de84d81 2024-11-26T14:31:42,690 INFO [StoreOpener-1c802a034a40c9c3ec031e1f2de84d81-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1c802a034a40c9c3ec031e1f2de84d81 columnFamilyName b 2024-11-26T14:31:42,690 DEBUG [StoreOpener-1c802a034a40c9c3ec031e1f2de84d81-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:31:42,690 INFO [StoreOpener-1c802a034a40c9c3ec031e1f2de84d81-1 {}] regionserver.HStore(327): Store=1c802a034a40c9c3ec031e1f2de84d81/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-26T14:31:42,691 INFO [StoreOpener-1c802a034a40c9c3ec031e1f2de84d81-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 1c802a034a40c9c3ec031e1f2de84d81 2024-11-26T14:31:42,692 INFO [StoreOpener-1c802a034a40c9c3ec031e1f2de84d81-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1c802a034a40c9c3ec031e1f2de84d81 columnFamilyName c 2024-11-26T14:31:42,692 DEBUG [StoreOpener-1c802a034a40c9c3ec031e1f2de84d81-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:31:42,692 INFO [StoreOpener-1c802a034a40c9c3ec031e1f2de84d81-1 {}] regionserver.HStore(327): Store=1c802a034a40c9c3ec031e1f2de84d81/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-26T14:31:42,692 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 1c802a034a40c9c3ec031e1f2de84d81 2024-11-26T14:31:42,693 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44999/hbase/data/default/testReplayEditsWrittenViaHRegion/1c802a034a40c9c3ec031e1f2de84d81 2024-11-26T14:31:42,694 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44999/hbase/data/default/testReplayEditsWrittenViaHRegion/1c802a034a40c9c3ec031e1f2de84d81 2024-11-26T14:31:42,695 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 1c802a034a40c9c3ec031e1f2de84d81 2024-11-26T14:31:42,695 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 1c802a034a40c9c3ec031e1f2de84d81 2024-11-26T14:31:42,696 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsWrittenViaHRegion descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-26T14:31:42,697 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 1c802a034a40c9c3ec031e1f2de84d81 2024-11-26T14:31:42,700 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44999/hbase/data/default/testReplayEditsWrittenViaHRegion/1c802a034a40c9c3ec031e1f2de84d81/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-26T14:31:42,701 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 1c802a034a40c9c3ec031e1f2de84d81; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=73779828, jitterRate=0.09940510988235474}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-26T14:31:42,701 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 1c802a034a40c9c3ec031e1f2de84d81: Writing region info on filesystem at 1732631502685Initializing all the Stores at 1732631502686 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732631502686Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732631502686Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732631502686Cleaning up temporary data from old regions at 1732631502695 (+9 ms)Region opened successfully at 1732631502701 (+6 ms) 2024-11-26T14:31:42,701 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 1c802a034a40c9c3ec031e1f2de84d81, disabling compactions & flushes 2024-11-26T14:31:42,701 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsWrittenViaHRegion,,1732631502657.1c802a034a40c9c3ec031e1f2de84d81. 2024-11-26T14:31:42,701 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsWrittenViaHRegion,,1732631502657.1c802a034a40c9c3ec031e1f2de84d81. 2024-11-26T14:31:42,701 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsWrittenViaHRegion,,1732631502657.1c802a034a40c9c3ec031e1f2de84d81. after waiting 0 ms 2024-11-26T14:31:42,701 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsWrittenViaHRegion,,1732631502657.1c802a034a40c9c3ec031e1f2de84d81. 2024-11-26T14:31:42,702 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsWrittenViaHRegion,,1732631502657.1c802a034a40c9c3ec031e1f2de84d81. 2024-11-26T14:31:42,702 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 1c802a034a40c9c3ec031e1f2de84d81: Waiting for close lock at 1732631502701Disabling compacts and flushes for region at 1732631502701Disabling writes for close at 1732631502701Writing region close event to WAL at 1732631502702 (+1 ms)Closed at 1732631502702 2024-11-26T14:31:42,704 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36307 is added to blk_1073741925_1104 (size=93) 2024-11-26T14:31:42,704 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38147 is added to blk_1073741925_1104 (size=93) 2024-11-26T14:31:42,704 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36277 is added to blk_1073741925_1104 (size=93) 2024-11-26T14:31:42,706 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-11-26T14:31:42,706 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-71861647:(num 1732631502661) 2024-11-26T14:31:42,706 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-26T14:31:42,707 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:44999/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732631502656, archiveDir=hdfs://localhost:44999/hbase/oldWALs, maxLogs=32 2024-11-26T14:31:42,723 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732631502656/wal.1732631502708, exclude list is [], retry=0 2024-11-26T14:31:42,726 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36277,DS-9cb9383d-a6c8-4ba6-83b3-6706331a6539,DISK] 2024-11-26T14:31:42,726 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38147,DS-44b12d44-38d0-4c76-b56f-3ca8d8caa398,DISK] 2024-11-26T14:31:42,727 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36307,DS-dd37de78-b3d5-4d7c-8223-4b15101c21bf,DISK] 2024-11-26T14:31:42,728 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732631502656/wal.1732631502708 2024-11-26T14:31:42,729 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:40449:40449),(127.0.0.1/127.0.0.1:35081:35081),(127.0.0.1/127.0.0.1:34431:34431)] 2024-11-26T14:31:42,729 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1c802a034a40c9c3ec031e1f2de84d81, NAME => 'testReplayEditsWrittenViaHRegion,,1732631502657.1c802a034a40c9c3ec031e1f2de84d81.', STARTKEY => '', ENDKEY => ''} 2024-11-26T14:31:42,729 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenViaHRegion,,1732631502657.1c802a034a40c9c3ec031e1f2de84d81.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-26T14:31:42,729 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 1c802a034a40c9c3ec031e1f2de84d81 2024-11-26T14:31:42,729 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 1c802a034a40c9c3ec031e1f2de84d81 2024-11-26T14:31:42,731 INFO [StoreOpener-1c802a034a40c9c3ec031e1f2de84d81-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 1c802a034a40c9c3ec031e1f2de84d81 2024-11-26T14:31:42,731 INFO [StoreOpener-1c802a034a40c9c3ec031e1f2de84d81-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1c802a034a40c9c3ec031e1f2de84d81 columnFamilyName a 2024-11-26T14:31:42,732 DEBUG [StoreOpener-1c802a034a40c9c3ec031e1f2de84d81-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:31:42,732 INFO [StoreOpener-1c802a034a40c9c3ec031e1f2de84d81-1 {}] regionserver.HStore(327): Store=1c802a034a40c9c3ec031e1f2de84d81/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-26T14:31:42,732 INFO [StoreOpener-1c802a034a40c9c3ec031e1f2de84d81-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 1c802a034a40c9c3ec031e1f2de84d81 2024-11-26T14:31:42,733 INFO [StoreOpener-1c802a034a40c9c3ec031e1f2de84d81-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1c802a034a40c9c3ec031e1f2de84d81 columnFamilyName b 2024-11-26T14:31:42,733 DEBUG [StoreOpener-1c802a034a40c9c3ec031e1f2de84d81-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:31:42,733 INFO [StoreOpener-1c802a034a40c9c3ec031e1f2de84d81-1 {}] regionserver.HStore(327): Store=1c802a034a40c9c3ec031e1f2de84d81/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-26T14:31:42,733 INFO [StoreOpener-1c802a034a40c9c3ec031e1f2de84d81-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 1c802a034a40c9c3ec031e1f2de84d81 2024-11-26T14:31:42,734 INFO [StoreOpener-1c802a034a40c9c3ec031e1f2de84d81-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1c802a034a40c9c3ec031e1f2de84d81 columnFamilyName c 2024-11-26T14:31:42,734 DEBUG [StoreOpener-1c802a034a40c9c3ec031e1f2de84d81-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:31:42,735 INFO [StoreOpener-1c802a034a40c9c3ec031e1f2de84d81-1 {}] regionserver.HStore(327): Store=1c802a034a40c9c3ec031e1f2de84d81/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-26T14:31:42,735 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 1c802a034a40c9c3ec031e1f2de84d81 2024-11-26T14:31:42,735 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44999/hbase/data/default/testReplayEditsWrittenViaHRegion/1c802a034a40c9c3ec031e1f2de84d81 2024-11-26T14:31:42,736 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44999/hbase/data/default/testReplayEditsWrittenViaHRegion/1c802a034a40c9c3ec031e1f2de84d81 2024-11-26T14:31:42,737 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 1c802a034a40c9c3ec031e1f2de84d81 2024-11-26T14:31:42,737 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 1c802a034a40c9c3ec031e1f2de84d81 2024-11-26T14:31:42,738 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsWrittenViaHRegion descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-26T14:31:42,739 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 1c802a034a40c9c3ec031e1f2de84d81 2024-11-26T14:31:42,740 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 1c802a034a40c9c3ec031e1f2de84d81; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=62821968, jitterRate=-0.06387972831726074}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-26T14:31:42,740 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 1c802a034a40c9c3ec031e1f2de84d81: Writing region info on filesystem at 1732631502729Initializing all the Stores at 1732631502730 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732631502730Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732631502731 (+1 ms)Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732631502731Cleaning up temporary data from old regions at 1732631502737 (+6 ms)Region opened successfully at 1732631502740 (+3 ms) 2024-11-26T14:31:42,747 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 1c802a034a40c9c3ec031e1f2de84d81 3/3 column families, dataSize=870 B heapSize=2.31 KB 2024-11-26T14:31:42,761 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44999/hbase/data/default/testReplayEditsWrittenViaHRegion/1c802a034a40c9c3ec031e1f2de84d81/.tmp/a/9c01c73227d7452cb3d90bcdb4c10d73 is 91, key is testReplayEditsWrittenViaHRegion/a:x0/1732631502740/Put/seqid=0 2024-11-26T14:31:42,767 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36307 is added to blk_1073741928_1107 (size=5958) 2024-11-26T14:31:42,767 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36277 is added to blk_1073741928_1107 (size=5958) 2024-11-26T14:31:42,768 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38147 is added to blk_1073741928_1107 (size=5958) 2024-11-26T14:31:42,768 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:44999/hbase/data/default/testReplayEditsWrittenViaHRegion/1c802a034a40c9c3ec031e1f2de84d81/.tmp/a/9c01c73227d7452cb3d90bcdb4c10d73 2024-11-26T14:31:42,773 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44999/hbase/data/default/testReplayEditsWrittenViaHRegion/1c802a034a40c9c3ec031e1f2de84d81/.tmp/a/9c01c73227d7452cb3d90bcdb4c10d73 as hdfs://localhost:44999/hbase/data/default/testReplayEditsWrittenViaHRegion/1c802a034a40c9c3ec031e1f2de84d81/a/9c01c73227d7452cb3d90bcdb4c10d73 2024-11-26T14:31:42,777 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44999/hbase/data/default/testReplayEditsWrittenViaHRegion/1c802a034a40c9c3ec031e1f2de84d81/a/9c01c73227d7452cb3d90bcdb4c10d73, entries=10, sequenceid=13, filesize=5.8 K 2024-11-26T14:31:42,779 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~870 B/870, heapSize ~1.80 KB/1840, currentSize=0 B/0 for 1c802a034a40c9c3ec031e1f2de84d81 in 31ms, sequenceid=13, compaction requested=false 2024-11-26T14:31:42,779 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for 1c802a034a40c9c3ec031e1f2de84d81: 2024-11-26T14:31:42,794 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 1c802a034a40c9c3ec031e1f2de84d81, disabling compactions & flushes 2024-11-26T14:31:42,794 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsWrittenViaHRegion,,1732631502657.1c802a034a40c9c3ec031e1f2de84d81. 2024-11-26T14:31:42,794 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsWrittenViaHRegion,,1732631502657.1c802a034a40c9c3ec031e1f2de84d81. 2024-11-26T14:31:42,794 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsWrittenViaHRegion,,1732631502657.1c802a034a40c9c3ec031e1f2de84d81. after waiting 0 ms 2024-11-26T14:31:42,794 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsWrittenViaHRegion,,1732631502657.1c802a034a40c9c3ec031e1f2de84d81. 2024-11-26T14:31:42,795 ERROR [Time-limited test {}] regionserver.HRegion(1960): Memstore data size is 1740 in region testReplayEditsWrittenViaHRegion,,1732631502657.1c802a034a40c9c3ec031e1f2de84d81. 2024-11-26T14:31:42,795 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsWrittenViaHRegion,,1732631502657.1c802a034a40c9c3ec031e1f2de84d81. 2024-11-26T14:31:42,795 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 1c802a034a40c9c3ec031e1f2de84d81: Waiting for close lock at 1732631502794Disabling compacts and flushes for region at 1732631502794Disabling writes for close at 1732631502794Writing region close event to WAL at 1732631502795 (+1 ms)Closed at 1732631502795 2024-11-26T14:31:42,797 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36307 is added to blk_1073741927_1106 (size=2805) 2024-11-26T14:31:42,797 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38147 is added to blk_1073741927_1106 (size=2805) 2024-11-26T14:31:42,797 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36277 is added to blk_1073741927_1106 (size=2805) 2024-11-26T14:31:42,806 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-26T14:31:42,814 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:44999/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732631502656/wal.1732631502708, size=2.7 K (2805bytes) 2024-11-26T14:31:42,814 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:44999/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732631502656/wal.1732631502708 2024-11-26T14:31:42,815 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:44999/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732631502656/wal.1732631502708 after 1ms 2024-11-26T14:31:42,817 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:44999/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732631502656/wal.1732631502708: isRecoveredEdits=false, hasTagCompression=true, hasValueCompression=false, valueCompressionType=null 2024-11-26T14:31:42,818 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:44999/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732631502656/wal.1732631502708 took 4ms 2024-11-26T14:31:42,820 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:44999/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732631502656/wal.1732631502708 so closing down 2024-11-26T14:31:42,820 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-26T14:31:42,821 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000003-wal.1732631502708.temp 2024-11-26T14:31:42,822 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:44999/hbase/data/default/testReplayEditsWrittenViaHRegion/1c802a034a40c9c3ec031e1f2de84d81/recovered.edits/0000000000000000003-wal.1732631502708.temp 2024-11-26T14:31:42,823 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-11-26T14:31:42,834 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36307 is added to blk_1073741929_1108 (size=2312) 2024-11-26T14:31:42,834 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36277 is added to blk_1073741929_1108 (size=2312) 2024-11-26T14:31:42,834 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38147 is added to blk_1073741929_1108 (size=2312) 2024-11-26T14:31:42,835 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:44999/hbase/data/default/testReplayEditsWrittenViaHRegion/1c802a034a40c9c3ec031e1f2de84d81/recovered.edits/0000000000000000003-wal.1732631502708.temp (wrote 30 edits, skipped 0 edits in 0 ms) 2024-11-26T14:31:42,836 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:44999/hbase/data/default/testReplayEditsWrittenViaHRegion/1c802a034a40c9c3ec031e1f2de84d81/recovered.edits/0000000000000000003-wal.1732631502708.temp to hdfs://localhost:44999/hbase/data/default/testReplayEditsWrittenViaHRegion/1c802a034a40c9c3ec031e1f2de84d81/recovered.edits/0000000000000000035 2024-11-26T14:31:42,837 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 32 edits across 1 Regions in 18 ms; skipped=2; WAL=hdfs://localhost:44999/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732631502656/wal.1732631502708, size=2.7 K, length=2805, corrupted=false, cancelled=false 2024-11-26T14:31:42,837 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:44999/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732631502656/wal.1732631502708, journal: Splitting hdfs://localhost:44999/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732631502656/wal.1732631502708, size=2.7 K (2805bytes) at 1732631502814Finishing writing output for hdfs://localhost:44999/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732631502656/wal.1732631502708 so closing down at 1732631502820 (+6 ms)Creating recovered edits writer path=hdfs://localhost:44999/hbase/data/default/testReplayEditsWrittenViaHRegion/1c802a034a40c9c3ec031e1f2de84d81/recovered.edits/0000000000000000003-wal.1732631502708.temp at 1732631502822 (+2 ms)3 split writer threads finished at 1732631502823 (+1 ms)Closed recovered edits writer path=hdfs://localhost:44999/hbase/data/default/testReplayEditsWrittenViaHRegion/1c802a034a40c9c3ec031e1f2de84d81/recovered.edits/0000000000000000003-wal.1732631502708.temp (wrote 30 edits, skipped 0 edits in 0 ms) at 1732631502835 (+12 ms)Rename recovered edits hdfs://localhost:44999/hbase/data/default/testReplayEditsWrittenViaHRegion/1c802a034a40c9c3ec031e1f2de84d81/recovered.edits/0000000000000000003-wal.1732631502708.temp to hdfs://localhost:44999/hbase/data/default/testReplayEditsWrittenViaHRegion/1c802a034a40c9c3ec031e1f2de84d81/recovered.edits/0000000000000000035 at 1732631502836 (+1 ms)Processed 32 edits across 1 Regions in 18 ms; skipped=2; WAL=hdfs://localhost:44999/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732631502656/wal.1732631502708, size=2.7 K, length=2805, corrupted=false, cancelled=false at 1732631502837 (+1 ms) 2024-11-26T14:31:42,838 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:44999/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732631502656/wal.1732631502708 to hdfs://localhost:44999/hbase/oldWALs/wal.1732631502708 2024-11-26T14:31:42,839 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:44999/hbase/data/default/testReplayEditsWrittenViaHRegion/1c802a034a40c9c3ec031e1f2de84d81/recovered.edits/0000000000000000035 2024-11-26T14:31:42,840 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-26T14:31:42,842 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:44999/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732631502656, archiveDir=hdfs://localhost:44999/hbase/oldWALs, maxLogs=32 2024-11-26T14:31:42,859 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732631502656/wal.1732631502842, exclude list is [], retry=0 2024-11-26T14:31:42,862 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36307,DS-dd37de78-b3d5-4d7c-8223-4b15101c21bf,DISK] 2024-11-26T14:31:42,863 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38147,DS-44b12d44-38d0-4c76-b56f-3ca8d8caa398,DISK] 2024-11-26T14:31:42,863 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36277,DS-9cb9383d-a6c8-4ba6-83b3-6706331a6539,DISK] 2024-11-26T14:31:42,865 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732631502656/wal.1732631502842 2024-11-26T14:31:42,865 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:34431:34431),(127.0.0.1/127.0.0.1:35081:35081),(127.0.0.1/127.0.0.1:40449:40449)] 2024-11-26T14:31:42,865 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1c802a034a40c9c3ec031e1f2de84d81, NAME => 'testReplayEditsWrittenViaHRegion,,1732631502657.1c802a034a40c9c3ec031e1f2de84d81.', STARTKEY => '', ENDKEY => ''} 2024-11-26T14:31:42,865 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenViaHRegion,,1732631502657.1c802a034a40c9c3ec031e1f2de84d81.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-26T14:31:42,865 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 1c802a034a40c9c3ec031e1f2de84d81 2024-11-26T14:31:42,865 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 1c802a034a40c9c3ec031e1f2de84d81 2024-11-26T14:31:42,866 INFO [StoreOpener-1c802a034a40c9c3ec031e1f2de84d81-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 1c802a034a40c9c3ec031e1f2de84d81 2024-11-26T14:31:42,867 INFO [StoreOpener-1c802a034a40c9c3ec031e1f2de84d81-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1c802a034a40c9c3ec031e1f2de84d81 columnFamilyName a 2024-11-26T14:31:42,867 DEBUG [StoreOpener-1c802a034a40c9c3ec031e1f2de84d81-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:31:42,872 DEBUG [StoreOpener-1c802a034a40c9c3ec031e1f2de84d81-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:44999/hbase/data/default/testReplayEditsWrittenViaHRegion/1c802a034a40c9c3ec031e1f2de84d81/a/9c01c73227d7452cb3d90bcdb4c10d73 2024-11-26T14:31:42,872 INFO [StoreOpener-1c802a034a40c9c3ec031e1f2de84d81-1 {}] regionserver.HStore(327): Store=1c802a034a40c9c3ec031e1f2de84d81/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-26T14:31:42,872 INFO [StoreOpener-1c802a034a40c9c3ec031e1f2de84d81-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 1c802a034a40c9c3ec031e1f2de84d81 2024-11-26T14:31:42,873 INFO [StoreOpener-1c802a034a40c9c3ec031e1f2de84d81-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1c802a034a40c9c3ec031e1f2de84d81 columnFamilyName b 2024-11-26T14:31:42,873 DEBUG [StoreOpener-1c802a034a40c9c3ec031e1f2de84d81-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:31:42,874 INFO [StoreOpener-1c802a034a40c9c3ec031e1f2de84d81-1 {}] regionserver.HStore(327): Store=1c802a034a40c9c3ec031e1f2de84d81/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-26T14:31:42,874 INFO [StoreOpener-1c802a034a40c9c3ec031e1f2de84d81-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 1c802a034a40c9c3ec031e1f2de84d81 2024-11-26T14:31:42,874 INFO [StoreOpener-1c802a034a40c9c3ec031e1f2de84d81-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1c802a034a40c9c3ec031e1f2de84d81 columnFamilyName c 2024-11-26T14:31:42,874 DEBUG [StoreOpener-1c802a034a40c9c3ec031e1f2de84d81-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:31:42,875 INFO [StoreOpener-1c802a034a40c9c3ec031e1f2de84d81-1 {}] regionserver.HStore(327): Store=1c802a034a40c9c3ec031e1f2de84d81/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-26T14:31:42,875 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 1c802a034a40c9c3ec031e1f2de84d81 2024-11-26T14:31:42,875 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44999/hbase/data/default/testReplayEditsWrittenViaHRegion/1c802a034a40c9c3ec031e1f2de84d81 2024-11-26T14:31:42,877 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:44999/hbase/data/default/testReplayEditsWrittenViaHRegion/1c802a034a40c9c3ec031e1f2de84d81 2024-11-26T14:31:42,877 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:44999/hbase/data/default/testReplayEditsWrittenViaHRegion/1c802a034a40c9c3ec031e1f2de84d81/recovered.edits/0000000000000000035 2024-11-26T14:31:42,879 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:44999/hbase/data/default/testReplayEditsWrittenViaHRegion/1c802a034a40c9c3ec031e1f2de84d81/recovered.edits/0000000000000000035: isRecoveredEdits=true, hasTagCompression=true, hasValueCompression=false, valueCompressionType=null 2024-11-26T14:31:42,880 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 20, skipped 10, firstSequenceIdInLog=3, maxSequenceIdInLog=35, path=hdfs://localhost:44999/hbase/data/default/testReplayEditsWrittenViaHRegion/1c802a034a40c9c3ec031e1f2de84d81/recovered.edits/0000000000000000035 2024-11-26T14:31:42,881 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 1c802a034a40c9c3ec031e1f2de84d81 3/3 column families, dataSize=1.70 KB heapSize=3.88 KB 2024-11-26T14:31:42,897 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44999/hbase/data/default/testReplayEditsWrittenViaHRegion/1c802a034a40c9c3ec031e1f2de84d81/.tmp/b/7cc4ce51d2894e17b670d6e8d5eef855 is 91, key is testReplayEditsWrittenViaHRegion/b:x0/1732631502779/Put/seqid=0 2024-11-26T14:31:42,903 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36277 is added to blk_1073741931_1110 (size=5958) 2024-11-26T14:31:42,903 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36307 is added to blk_1073741931_1110 (size=5958) 2024-11-26T14:31:42,903 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38147 is added to blk_1073741931_1110 (size=5958) 2024-11-26T14:31:42,904 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=35 (bloomFilter=true), to=hdfs://localhost:44999/hbase/data/default/testReplayEditsWrittenViaHRegion/1c802a034a40c9c3ec031e1f2de84d81/.tmp/b/7cc4ce51d2894e17b670d6e8d5eef855 2024-11-26T14:31:42,922 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44999/hbase/data/default/testReplayEditsWrittenViaHRegion/1c802a034a40c9c3ec031e1f2de84d81/.tmp/c/64a78b8ef28a4edd854db882da5039a5 is 91, key is testReplayEditsWrittenViaHRegion/c:x0/1732631502785/Put/seqid=0 2024-11-26T14:31:42,928 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38147 is added to blk_1073741932_1111 (size=5958) 2024-11-26T14:31:42,928 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36277 is added to blk_1073741932_1111 (size=5958) 2024-11-26T14:31:42,928 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36307 is added to blk_1073741932_1111 (size=5958) 2024-11-26T14:31:42,929 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=35 (bloomFilter=true), to=hdfs://localhost:44999/hbase/data/default/testReplayEditsWrittenViaHRegion/1c802a034a40c9c3ec031e1f2de84d81/.tmp/c/64a78b8ef28a4edd854db882da5039a5 2024-11-26T14:31:42,933 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44999/hbase/data/default/testReplayEditsWrittenViaHRegion/1c802a034a40c9c3ec031e1f2de84d81/.tmp/b/7cc4ce51d2894e17b670d6e8d5eef855 as hdfs://localhost:44999/hbase/data/default/testReplayEditsWrittenViaHRegion/1c802a034a40c9c3ec031e1f2de84d81/b/7cc4ce51d2894e17b670d6e8d5eef855 2024-11-26T14:31:42,941 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44999/hbase/data/default/testReplayEditsWrittenViaHRegion/1c802a034a40c9c3ec031e1f2de84d81/b/7cc4ce51d2894e17b670d6e8d5eef855, entries=10, sequenceid=35, filesize=5.8 K 2024-11-26T14:31:42,942 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44999/hbase/data/default/testReplayEditsWrittenViaHRegion/1c802a034a40c9c3ec031e1f2de84d81/.tmp/c/64a78b8ef28a4edd854db882da5039a5 as hdfs://localhost:44999/hbase/data/default/testReplayEditsWrittenViaHRegion/1c802a034a40c9c3ec031e1f2de84d81/c/64a78b8ef28a4edd854db882da5039a5 2024-11-26T14:31:42,948 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44999/hbase/data/default/testReplayEditsWrittenViaHRegion/1c802a034a40c9c3ec031e1f2de84d81/c/64a78b8ef28a4edd854db882da5039a5, entries=10, sequenceid=35, filesize=5.8 K 2024-11-26T14:31:42,949 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~1.70 KB/1740, heapSize ~3.59 KB/3680, currentSize=0 B/0 for 1c802a034a40c9c3ec031e1f2de84d81 in 68ms, sequenceid=35, compaction requested=false; wal=null 2024-11-26T14:31:42,949 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:44999/hbase/data/default/testReplayEditsWrittenViaHRegion/1c802a034a40c9c3ec031e1f2de84d81/recovered.edits/0000000000000000035 2024-11-26T14:31:42,951 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 1c802a034a40c9c3ec031e1f2de84d81 2024-11-26T14:31:42,951 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 1c802a034a40c9c3ec031e1f2de84d81 2024-11-26T14:31:42,951 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsWrittenViaHRegion descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-26T14:31:42,953 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 1c802a034a40c9c3ec031e1f2de84d81 2024-11-26T14:31:42,955 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44999/hbase/data/default/testReplayEditsWrittenViaHRegion/1c802a034a40c9c3ec031e1f2de84d81/recovered.edits/35.seqid, newMaxSeqId=35, maxSeqId=1 2024-11-26T14:31:42,955 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 1c802a034a40c9c3ec031e1f2de84d81; next sequenceid=36; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=65511347, jitterRate=-0.023804858326911926}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-26T14:31:42,956 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 1c802a034a40c9c3ec031e1f2de84d81: Writing region info on filesystem at 1732631502865Initializing all the Stores at 1732631502866 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732631502866Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732631502866Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732631502866Obtaining lock to block concurrent updates at 1732631502881 (+15 ms)Preparing flush snapshotting stores in 1c802a034a40c9c3ec031e1f2de84d81 at 1732631502881Finished memstore snapshotting testReplayEditsWrittenViaHRegion,,1732631502657.1c802a034a40c9c3ec031e1f2de84d81., syncing WAL and waiting on mvcc, flushsize=dataSize=1740, getHeapSize=3920, getOffHeapSize=0, getCellsCount=20 at 1732631502881Flushing stores of testReplayEditsWrittenViaHRegion,,1732631502657.1c802a034a40c9c3ec031e1f2de84d81. at 1732631502881Flushing 1c802a034a40c9c3ec031e1f2de84d81/b: creating writer at 1732631502881Flushing 1c802a034a40c9c3ec031e1f2de84d81/b: appending metadata at 1732631502896 (+15 ms)Flushing 1c802a034a40c9c3ec031e1f2de84d81/b: closing flushed file at 1732631502896Flushing 1c802a034a40c9c3ec031e1f2de84d81/c: creating writer at 1732631502908 (+12 ms)Flushing 1c802a034a40c9c3ec031e1f2de84d81/c: appending metadata at 1732631502922 (+14 ms)Flushing 1c802a034a40c9c3ec031e1f2de84d81/c: closing flushed file at 1732631502922Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@289efc5d: reopening flushed file at 1732631502933 (+11 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@2ad84dc6: reopening flushed file at 1732631502941 (+8 ms)Finished flush of dataSize ~1.70 KB/1740, heapSize ~3.59 KB/3680, currentSize=0 B/0 for 1c802a034a40c9c3ec031e1f2de84d81 in 68ms, sequenceid=35, compaction requested=false; wal=null at 1732631502949 (+8 ms)Cleaning up temporary data from old regions at 1732631502951 (+2 ms)Region opened successfully at 1732631502956 (+5 ms) 2024-11-26T14:31:43,019 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:44999/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732631502656/wal.1732631502842, size=0 (0bytes) 2024-11-26T14:31:43,020 WARN [Time-limited test {}] wal.WALSplitter(453): File hdfs://localhost:44999/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732631502656/wal.1732631502842 might be still open, length is 0 2024-11-26T14:31:43,020 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:44999/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732631502656/wal.1732631502842 2024-11-26T14:31:43,020 WARN [IPC Server handler 3 on default port 44999 {}] namenode.FSNamesystem(3854): DIR* NameSystem.internalReleaseLease: File /hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732631502656/wal.1732631502842 has not been closed. Lease recovery is in progress. RecoveryId = 1112 for block blk_1073741930_1109 2024-11-26T14:31:43,020 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=0 on file=hdfs://localhost:44999/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732631502656/wal.1732631502842 after 0ms 2024-11-26T14:31:43,849 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_2043263076_22 at /127.0.0.1:47446 [Receiving block BP-1347678837-172.17.0.3-1732631464290:blk_1073741930_1109] {}] datanode.DataXceiver(331): 127.0.0.1:36307:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:47446 dst: /127.0.0.1:36307 java.io.InterruptedIOException: Interrupted while waiting for IO on channel java.nio.channels.SocketChannel[connected local=localhost/127.0.0.1:36307 remote=/127.0.0.1:47446]. Total timeout mills is 60000, 59133 millis timeout left. at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:350) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?] at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?] at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?] at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?] at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?] at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T14:31:43,850 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_2043263076_22 at /127.0.0.1:43036 [Receiving block BP-1347678837-172.17.0.3-1732631464290:blk_1073741930_1109] {}] datanode.DataXceiver(331): 127.0.0.1:38147:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:43036 dst: /127.0.0.1:38147 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T14:31:43,850 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_2043263076_22 at /127.0.0.1:39006 [Receiving block BP-1347678837-172.17.0.3-1732631464290:blk_1073741930_1109] {}] datanode.DataXceiver(331): 127.0.0.1:36277:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:39006 dst: /127.0.0.1:36277 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T14:31:43,890 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36307 is added to blk_1073741930_1112 (size=2304) 2024-11-26T14:31:43,891 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38147 is added to blk_1073741930_1112 (size=2304) 2024-11-26T14:31:47,022 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=1 on file=hdfs://localhost:44999/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732631502656/wal.1732631502842 after 4002ms 2024-11-26T14:31:47,031 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:44999/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732631502656/wal.1732631502842: isRecoveredEdits=false, hasTagCompression=true, hasValueCompression=false, valueCompressionType=null 2024-11-26T14:31:47,031 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:44999/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732631502656/wal.1732631502842 took 4012ms 2024-11-26T14:31:47,034 INFO [Time-limited test {}] wal.WALSplitter(493): EOF from hdfs://localhost:44999/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732631502656/wal.1732631502842; continuing. 2024-11-26T14:31:47,034 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:44999/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732631502656/wal.1732631502842 so closing down 2024-11-26T14:31:47,034 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-26T14:31:47,036 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000037-wal.1732631502842.temp 2024-11-26T14:31:47,038 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:44999/hbase/data/default/testReplayEditsWrittenViaHRegion/1c802a034a40c9c3ec031e1f2de84d81/recovered.edits/0000000000000000037-wal.1732631502842.temp 2024-11-26T14:31:47,038 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-11-26T14:31:47,054 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38147 is added to blk_1073741933_1113 (size=2312) 2024-11-26T14:31:47,054 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36307 is added to blk_1073741933_1113 (size=2312) 2024-11-26T14:31:47,054 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36277 is added to blk_1073741933_1113 (size=2312) 2024-11-26T14:31:47,055 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:44999/hbase/data/default/testReplayEditsWrittenViaHRegion/1c802a034a40c9c3ec031e1f2de84d81/recovered.edits/0000000000000000037-wal.1732631502842.temp (wrote 30 edits, skipped 0 edits in 0 ms) 2024-11-26T14:31:47,056 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:44999/hbase/data/default/testReplayEditsWrittenViaHRegion/1c802a034a40c9c3ec031e1f2de84d81/recovered.edits/0000000000000000037-wal.1732631502842.temp to hdfs://localhost:44999/hbase/data/default/testReplayEditsWrittenViaHRegion/1c802a034a40c9c3ec031e1f2de84d81/recovered.edits/0000000000000000066 2024-11-26T14:31:47,056 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 30 edits across 1 Regions in 24 ms; skipped=0; WAL=hdfs://localhost:44999/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732631502656/wal.1732631502842, size=0, length=0, corrupted=false, cancelled=false 2024-11-26T14:31:47,056 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:44999/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732631502656/wal.1732631502842, journal: Splitting hdfs://localhost:44999/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732631502656/wal.1732631502842, size=0 (0bytes) at 1732631503019Finishing writing output for hdfs://localhost:44999/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732631502656/wal.1732631502842 so closing down at 1732631507034 (+4015 ms)Creating recovered edits writer path=hdfs://localhost:44999/hbase/data/default/testReplayEditsWrittenViaHRegion/1c802a034a40c9c3ec031e1f2de84d81/recovered.edits/0000000000000000037-wal.1732631502842.temp at 1732631507038 (+4 ms)3 split writer threads finished at 1732631507038Closed recovered edits writer path=hdfs://localhost:44999/hbase/data/default/testReplayEditsWrittenViaHRegion/1c802a034a40c9c3ec031e1f2de84d81/recovered.edits/0000000000000000037-wal.1732631502842.temp (wrote 30 edits, skipped 0 edits in 0 ms) at 1732631507055 (+17 ms)Rename recovered edits hdfs://localhost:44999/hbase/data/default/testReplayEditsWrittenViaHRegion/1c802a034a40c9c3ec031e1f2de84d81/recovered.edits/0000000000000000037-wal.1732631502842.temp to hdfs://localhost:44999/hbase/data/default/testReplayEditsWrittenViaHRegion/1c802a034a40c9c3ec031e1f2de84d81/recovered.edits/0000000000000000066 at 1732631507056 (+1 ms)Processed 30 edits across 1 Regions in 24 ms; skipped=0; WAL=hdfs://localhost:44999/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732631502656/wal.1732631502842, size=0, length=0, corrupted=false, cancelled=false at 1732631507056 2024-11-26T14:31:47,058 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:44999/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732631502656/wal.1732631502842 to hdfs://localhost:44999/hbase/oldWALs/wal.1732631502842 2024-11-26T14:31:47,058 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:44999/hbase/data/default/testReplayEditsWrittenViaHRegion/1c802a034a40c9c3ec031e1f2de84d81/recovered.edits/0000000000000000066 2024-11-26T14:31:47,059 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-26T14:31:47,060 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:44999/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732631502656, archiveDir=hdfs://localhost:44999/hbase/oldWALs, maxLogs=32 2024-11-26T14:31:47,073 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732631502656/wal.1732631507061, exclude list is [], retry=0 2024-11-26T14:31:47,075 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36277,DS-9cb9383d-a6c8-4ba6-83b3-6706331a6539,DISK] 2024-11-26T14:31:47,075 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38147,DS-44b12d44-38d0-4c76-b56f-3ca8d8caa398,DISK] 2024-11-26T14:31:47,076 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36307,DS-dd37de78-b3d5-4d7c-8223-4b15101c21bf,DISK] 2024-11-26T14:31:47,077 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732631502656/wal.1732631507061 2024-11-26T14:31:47,077 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:40449:40449),(127.0.0.1/127.0.0.1:35081:35081),(127.0.0.1/127.0.0.1:34431:34431)] 2024-11-26T14:31:47,077 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenViaHRegion,,1732631502657.1c802a034a40c9c3ec031e1f2de84d81.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-26T14:31:47,079 INFO [StoreOpener-1c802a034a40c9c3ec031e1f2de84d81-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 1c802a034a40c9c3ec031e1f2de84d81 2024-11-26T14:31:47,079 INFO [StoreOpener-1c802a034a40c9c3ec031e1f2de84d81-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1c802a034a40c9c3ec031e1f2de84d81 columnFamilyName a 2024-11-26T14:31:47,080 DEBUG [StoreOpener-1c802a034a40c9c3ec031e1f2de84d81-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:31:47,084 DEBUG [StoreOpener-1c802a034a40c9c3ec031e1f2de84d81-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:44999/hbase/data/default/testReplayEditsWrittenViaHRegion/1c802a034a40c9c3ec031e1f2de84d81/a/9c01c73227d7452cb3d90bcdb4c10d73 2024-11-26T14:31:47,085 INFO [StoreOpener-1c802a034a40c9c3ec031e1f2de84d81-1 {}] regionserver.HStore(327): Store=1c802a034a40c9c3ec031e1f2de84d81/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-26T14:31:47,085 INFO [StoreOpener-1c802a034a40c9c3ec031e1f2de84d81-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 1c802a034a40c9c3ec031e1f2de84d81 2024-11-26T14:31:47,085 INFO [StoreOpener-1c802a034a40c9c3ec031e1f2de84d81-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1c802a034a40c9c3ec031e1f2de84d81 columnFamilyName b 2024-11-26T14:31:47,085 DEBUG [StoreOpener-1c802a034a40c9c3ec031e1f2de84d81-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:31:47,090 DEBUG [StoreOpener-1c802a034a40c9c3ec031e1f2de84d81-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:44999/hbase/data/default/testReplayEditsWrittenViaHRegion/1c802a034a40c9c3ec031e1f2de84d81/b/7cc4ce51d2894e17b670d6e8d5eef855 2024-11-26T14:31:47,091 INFO [StoreOpener-1c802a034a40c9c3ec031e1f2de84d81-1 {}] regionserver.HStore(327): Store=1c802a034a40c9c3ec031e1f2de84d81/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-26T14:31:47,091 INFO [StoreOpener-1c802a034a40c9c3ec031e1f2de84d81-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 1c802a034a40c9c3ec031e1f2de84d81 2024-11-26T14:31:47,092 INFO [StoreOpener-1c802a034a40c9c3ec031e1f2de84d81-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1c802a034a40c9c3ec031e1f2de84d81 columnFamilyName c 2024-11-26T14:31:47,092 DEBUG [StoreOpener-1c802a034a40c9c3ec031e1f2de84d81-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T14:31:47,098 DEBUG [StoreOpener-1c802a034a40c9c3ec031e1f2de84d81-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:44999/hbase/data/default/testReplayEditsWrittenViaHRegion/1c802a034a40c9c3ec031e1f2de84d81/c/64a78b8ef28a4edd854db882da5039a5 2024-11-26T14:31:47,098 INFO [StoreOpener-1c802a034a40c9c3ec031e1f2de84d81-1 {}] regionserver.HStore(327): Store=1c802a034a40c9c3ec031e1f2de84d81/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-26T14:31:47,098 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 1c802a034a40c9c3ec031e1f2de84d81 2024-11-26T14:31:47,099 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44999/hbase/data/default/testReplayEditsWrittenViaHRegion/1c802a034a40c9c3ec031e1f2de84d81 2024-11-26T14:31:47,101 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:44999/hbase/data/default/testReplayEditsWrittenViaHRegion/1c802a034a40c9c3ec031e1f2de84d81 2024-11-26T14:31:47,102 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:44999/hbase/data/default/testReplayEditsWrittenViaHRegion/1c802a034a40c9c3ec031e1f2de84d81/recovered.edits/0000000000000000066 2024-11-26T14:31:47,104 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:44999/hbase/data/default/testReplayEditsWrittenViaHRegion/1c802a034a40c9c3ec031e1f2de84d81/recovered.edits/0000000000000000066: isRecoveredEdits=true, hasTagCompression=true, hasValueCompression=false, valueCompressionType=null 2024-11-26T14:31:47,107 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 30, skipped 0, firstSequenceIdInLog=37, maxSequenceIdInLog=66, path=hdfs://localhost:44999/hbase/data/default/testReplayEditsWrittenViaHRegion/1c802a034a40c9c3ec031e1f2de84d81/recovered.edits/0000000000000000066 2024-11-26T14:31:47,107 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 1c802a034a40c9c3ec031e1f2de84d81 3/3 column families, dataSize=2.55 KB heapSize=5.44 KB 2024-11-26T14:31:47,121 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44999/hbase/data/default/testReplayEditsWrittenViaHRegion/1c802a034a40c9c3ec031e1f2de84d81/.tmp/a/3dc1c8bf418242748d4be0741cdc5fb4 is 91, key is testReplayEditsWrittenViaHRegion/a:y0/1732631502963/Put/seqid=0 2024-11-26T14:31:47,126 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36307 is added to blk_1073741935_1115 (size=5958) 2024-11-26T14:31:47,126 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36277 is added to blk_1073741935_1115 (size=5958) 2024-11-26T14:31:47,127 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38147 is added to blk_1073741935_1115 (size=5958) 2024-11-26T14:31:47,127 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=66 (bloomFilter=true), to=hdfs://localhost:44999/hbase/data/default/testReplayEditsWrittenViaHRegion/1c802a034a40c9c3ec031e1f2de84d81/.tmp/a/3dc1c8bf418242748d4be0741cdc5fb4 2024-11-26T14:31:47,168 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44999/hbase/data/default/testReplayEditsWrittenViaHRegion/1c802a034a40c9c3ec031e1f2de84d81/.tmp/b/c24056ce321343799965ca029ac1179e is 91, key is testReplayEditsWrittenViaHRegion/b:y0/1732631502970/Put/seqid=0 2024-11-26T14:31:47,174 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36277 is added to blk_1073741936_1116 (size=5958) 2024-11-26T14:31:47,174 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36307 is added to blk_1073741936_1116 (size=5958) 2024-11-26T14:31:47,174 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38147 is added to blk_1073741936_1116 (size=5958) 2024-11-26T14:31:47,174 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=66 (bloomFilter=true), to=hdfs://localhost:44999/hbase/data/default/testReplayEditsWrittenViaHRegion/1c802a034a40c9c3ec031e1f2de84d81/.tmp/b/c24056ce321343799965ca029ac1179e 2024-11-26T14:31:47,192 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44999/hbase/data/default/testReplayEditsWrittenViaHRegion/1c802a034a40c9c3ec031e1f2de84d81/.tmp/c/35739fa50f1442a9a547da0c20fc0eb4 is 91, key is testReplayEditsWrittenViaHRegion/c:y0/1732631502976/Put/seqid=0 2024-11-26T14:31:47,197 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36277 is added to blk_1073741937_1117 (size=5958) 2024-11-26T14:31:47,198 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36307 is added to blk_1073741937_1117 (size=5958) 2024-11-26T14:31:47,198 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38147 is added to blk_1073741937_1117 (size=5958) 2024-11-26T14:31:47,198 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=66 (bloomFilter=true), to=hdfs://localhost:44999/hbase/data/default/testReplayEditsWrittenViaHRegion/1c802a034a40c9c3ec031e1f2de84d81/.tmp/c/35739fa50f1442a9a547da0c20fc0eb4 2024-11-26T14:31:47,203 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44999/hbase/data/default/testReplayEditsWrittenViaHRegion/1c802a034a40c9c3ec031e1f2de84d81/.tmp/a/3dc1c8bf418242748d4be0741cdc5fb4 as hdfs://localhost:44999/hbase/data/default/testReplayEditsWrittenViaHRegion/1c802a034a40c9c3ec031e1f2de84d81/a/3dc1c8bf418242748d4be0741cdc5fb4 2024-11-26T14:31:47,207 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44999/hbase/data/default/testReplayEditsWrittenViaHRegion/1c802a034a40c9c3ec031e1f2de84d81/a/3dc1c8bf418242748d4be0741cdc5fb4, entries=10, sequenceid=66, filesize=5.8 K 2024-11-26T14:31:47,209 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44999/hbase/data/default/testReplayEditsWrittenViaHRegion/1c802a034a40c9c3ec031e1f2de84d81/.tmp/b/c24056ce321343799965ca029ac1179e as hdfs://localhost:44999/hbase/data/default/testReplayEditsWrittenViaHRegion/1c802a034a40c9c3ec031e1f2de84d81/b/c24056ce321343799965ca029ac1179e 2024-11-26T14:31:47,214 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44999/hbase/data/default/testReplayEditsWrittenViaHRegion/1c802a034a40c9c3ec031e1f2de84d81/b/c24056ce321343799965ca029ac1179e, entries=10, sequenceid=66, filesize=5.8 K 2024-11-26T14:31:47,215 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44999/hbase/data/default/testReplayEditsWrittenViaHRegion/1c802a034a40c9c3ec031e1f2de84d81/.tmp/c/35739fa50f1442a9a547da0c20fc0eb4 as hdfs://localhost:44999/hbase/data/default/testReplayEditsWrittenViaHRegion/1c802a034a40c9c3ec031e1f2de84d81/c/35739fa50f1442a9a547da0c20fc0eb4 2024-11-26T14:31:47,219 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44999/hbase/data/default/testReplayEditsWrittenViaHRegion/1c802a034a40c9c3ec031e1f2de84d81/c/35739fa50f1442a9a547da0c20fc0eb4, entries=10, sequenceid=66, filesize=5.8 K 2024-11-26T14:31:47,220 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~2.55 KB/2610, heapSize ~5.39 KB/5520, currentSize=0 B/0 for 1c802a034a40c9c3ec031e1f2de84d81 in 112ms, sequenceid=66, compaction requested=false; wal=null 2024-11-26T14:31:47,220 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:44999/hbase/data/default/testReplayEditsWrittenViaHRegion/1c802a034a40c9c3ec031e1f2de84d81/recovered.edits/0000000000000000066 2024-11-26T14:31:47,221 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 1c802a034a40c9c3ec031e1f2de84d81 2024-11-26T14:31:47,221 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 1c802a034a40c9c3ec031e1f2de84d81 2024-11-26T14:31:47,222 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsWrittenViaHRegion descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-26T14:31:47,223 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 1c802a034a40c9c3ec031e1f2de84d81 2024-11-26T14:31:47,225 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44999/hbase/data/default/testReplayEditsWrittenViaHRegion/1c802a034a40c9c3ec031e1f2de84d81/recovered.edits/66.seqid, newMaxSeqId=66, maxSeqId=35 2024-11-26T14:31:47,226 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 1c802a034a40c9c3ec031e1f2de84d81; next sequenceid=67; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=72117047, jitterRate=0.07462774217128754}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-26T14:31:47,226 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 1c802a034a40c9c3ec031e1f2de84d81: Writing region info on filesystem at 1732631507078Initializing all the Stores at 1732631507078Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732631507078Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732631507079 (+1 ms)Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732631507079Obtaining lock to block concurrent updates at 1732631507107 (+28 ms)Preparing flush snapshotting stores in 1c802a034a40c9c3ec031e1f2de84d81 at 1732631507107Finished memstore snapshotting testReplayEditsWrittenViaHRegion,,1732631502657.1c802a034a40c9c3ec031e1f2de84d81., syncing WAL and waiting on mvcc, flushsize=dataSize=2610, getHeapSize=5520, getOffHeapSize=0, getCellsCount=30 at 1732631507107Flushing stores of testReplayEditsWrittenViaHRegion,,1732631502657.1c802a034a40c9c3ec031e1f2de84d81. at 1732631507107Flushing 1c802a034a40c9c3ec031e1f2de84d81/a: creating writer at 1732631507108 (+1 ms)Flushing 1c802a034a40c9c3ec031e1f2de84d81/a: appending metadata at 1732631507120 (+12 ms)Flushing 1c802a034a40c9c3ec031e1f2de84d81/a: closing flushed file at 1732631507120Flushing 1c802a034a40c9c3ec031e1f2de84d81/b: creating writer at 1732631507131 (+11 ms)Flushing 1c802a034a40c9c3ec031e1f2de84d81/b: appending metadata at 1732631507167 (+36 ms)Flushing 1c802a034a40c9c3ec031e1f2de84d81/b: closing flushed file at 1732631507167Flushing 1c802a034a40c9c3ec031e1f2de84d81/c: creating writer at 1732631507179 (+12 ms)Flushing 1c802a034a40c9c3ec031e1f2de84d81/c: appending metadata at 1732631507191 (+12 ms)Flushing 1c802a034a40c9c3ec031e1f2de84d81/c: closing flushed file at 1732631507192 (+1 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@6fe26336: reopening flushed file at 1732631507202 (+10 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@2760200f: reopening flushed file at 1732631507208 (+6 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@40047bc5: reopening flushed file at 1732631507214 (+6 ms)Finished flush of dataSize ~2.55 KB/2610, heapSize ~5.39 KB/5520, currentSize=0 B/0 for 1c802a034a40c9c3ec031e1f2de84d81 in 112ms, sequenceid=66, compaction requested=false; wal=null at 1732631507220 (+6 ms)Cleaning up temporary data from old regions at 1732631507221 (+1 ms)Region opened successfully at 1732631507226 (+5 ms) 2024-11-26T14:31:47,237 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 1c802a034a40c9c3ec031e1f2de84d81, disabling compactions & flushes 2024-11-26T14:31:47,237 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsWrittenViaHRegion,,1732631502657.1c802a034a40c9c3ec031e1f2de84d81. 2024-11-26T14:31:47,237 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsWrittenViaHRegion,,1732631502657.1c802a034a40c9c3ec031e1f2de84d81. 2024-11-26T14:31:47,237 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsWrittenViaHRegion,,1732631502657.1c802a034a40c9c3ec031e1f2de84d81. after waiting 0 ms 2024-11-26T14:31:47,237 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsWrittenViaHRegion,,1732631502657.1c802a034a40c9c3ec031e1f2de84d81. 2024-11-26T14:31:47,239 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsWrittenViaHRegion,,1732631502657.1c802a034a40c9c3ec031e1f2de84d81. 2024-11-26T14:31:47,239 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 1c802a034a40c9c3ec031e1f2de84d81: Waiting for close lock at 1732631507237Disabling compacts and flushes for region at 1732631507237Disabling writes for close at 1732631507237Writing region close event to WAL at 1732631507238 (+1 ms)Closed at 1732631507238 2024-11-26T14:31:47,241 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36307 is added to blk_1073741934_1114 (size=93) 2024-11-26T14:31:47,241 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36277 is added to blk_1073741934_1114 (size=93) 2024-11-26T14:31:47,241 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38147 is added to blk_1073741934_1114 (size=93) 2024-11-26T14:31:47,243 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-11-26T14:31:47,243 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL wal:(num 1732631507061) 2024-11-26T14:31:47,256 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestAsyncWALReplayCompressed#testReplayEditsWrittenViaHRegion Thread=451 (was 446) Potentially hanging thread: AsyncFSWAL-28-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Client (1987708253) connection to localhost/127.0.0.1:44999 from jenkinstestReplayEditsWrittenViaHRegion java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) Potentially hanging thread: AsyncFSWAL-28-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_250276958_22 at /127.0.0.1:39026 [Waiting for operation #25] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_250276958_22 at /127.0.0.1:43064 [Waiting for operation #10] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: LeaseRenewer:jenkinstestReplayEditsWrittenViaHRegion@localhost:44999 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_250276958_22 at /127.0.0.1:47510 [Waiting for operation #8] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-28-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=1414 (was 1348) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=239 (was 260), ProcessCount=11 (was 11), AvailableMemoryMB=8936 (was 8974) 2024-11-26T14:31:47,256 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1414 is superior to 1024 2024-11-26T14:31:47,256 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster 2024-11-26T14:31:47,256 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-26T14:31:47,257 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79) at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611) at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-26T14:31:47,257 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T14:31:47,257 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T14:31:47,257 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-26T14:31:47,257 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster 2024-11-26T14:31:47,258 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=1849544579, stopped=false 2024-11-26T14:31:47,258 INFO [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=19945b63c2b4,43577,1732631468672 2024-11-26T14:31:47,372 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40179-0x101778e13450002, quorum=127.0.0.1:49403, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-26T14:31:47,372 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43577-0x101778e13450000, quorum=127.0.0.1:49403, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-26T14:31:47,372 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37467-0x101778e13450003, quorum=127.0.0.1:49403, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-26T14:31:47,372 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40179-0x101778e13450002, quorum=127.0.0.1:49403, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-26T14:31:47,372 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43577-0x101778e13450000, quorum=127.0.0.1:49403, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-26T14:31:47,372 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-26T14:31:47,372 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37467-0x101778e13450003, quorum=127.0.0.1:49403, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-26T14:31:47,372 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-26T14:31:47,373 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277) at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265) at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416) at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-26T14:31:47,373 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T14:31:47,373 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '19945b63c2b4,40179,1732631469496' ***** 2024-11-26T14:31:47,373 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-11-26T14:31:47,373 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '19945b63c2b4,37467,1732631469538' ***** 2024-11-26T14:31:47,373 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-11-26T14:31:47,373 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:37467-0x101778e13450003, quorum=127.0.0.1:49403, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-26T14:31:47,373 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:43577-0x101778e13450000, quorum=127.0.0.1:49403, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-26T14:31:47,374 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:40179-0x101778e13450002, quorum=127.0.0.1:49403, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-26T14:31:47,374 INFO [RS:2;19945b63c2b4:37467 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-26T14:31:47,374 INFO [RS:1;19945b63c2b4:40179 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-26T14:31:47,374 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-26T14:31:47,374 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-26T14:31:47,374 INFO [RS:2;19945b63c2b4:37467 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-26T14:31:47,374 INFO [RS:2;19945b63c2b4:37467 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-26T14:31:47,375 INFO [RS:2;19945b63c2b4:37467 {}] regionserver.HRegionServer(3091): Received CLOSE for 40329a66ba4e990f8463ecffc5dfdaa9 2024-11-26T14:31:47,375 INFO [RS:1;19945b63c2b4:40179 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-26T14:31:47,375 INFO [RS:1;19945b63c2b4:40179 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-26T14:31:47,375 INFO [RS:1;19945b63c2b4:40179 {}] regionserver.HRegionServer(959): stopping server 19945b63c2b4,40179,1732631469496 2024-11-26T14:31:47,375 INFO [RS:2;19945b63c2b4:37467 {}] regionserver.HRegionServer(959): stopping server 19945b63c2b4,37467,1732631469538 2024-11-26T14:31:47,375 INFO [RS:1;19945b63c2b4:40179 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-26T14:31:47,375 INFO [RS:2;19945b63c2b4:37467 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-26T14:31:47,375 INFO [RS:1;19945b63c2b4:40179 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:1;19945b63c2b4:40179. 2024-11-26T14:31:47,375 INFO [RS:2;19945b63c2b4:37467 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:2;19945b63c2b4:37467. 2024-11-26T14:31:47,375 DEBUG [RS:1;19945b63c2b4:40179 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-26T14:31:47,375 DEBUG [RS_CLOSE_REGION-regionserver/19945b63c2b4:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 40329a66ba4e990f8463ecffc5dfdaa9, disabling compactions & flushes 2024-11-26T14:31:47,375 DEBUG [RS:2;19945b63c2b4:37467 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-26T14:31:47,375 DEBUG [RS:1;19945b63c2b4:40179 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T14:31:47,375 DEBUG [RS:2;19945b63c2b4:37467 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T14:31:47,375 INFO [RS_CLOSE_REGION-regionserver/19945b63c2b4:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testReplayEditsAfterRegionMovedWithMultiCF,,1732631484097.40329a66ba4e990f8463ecffc5dfdaa9. 2024-11-26T14:31:47,375 INFO [RS:1;19945b63c2b4:40179 {}] regionserver.HRegionServer(976): stopping server 19945b63c2b4,40179,1732631469496; all regions closed. 2024-11-26T14:31:47,375 DEBUG [RS_CLOSE_REGION-regionserver/19945b63c2b4:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1732631484097.40329a66ba4e990f8463ecffc5dfdaa9. 2024-11-26T14:31:47,376 INFO [RS:2;19945b63c2b4:37467 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-26T14:31:47,376 DEBUG [RS_CLOSE_REGION-regionserver/19945b63c2b4:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1732631484097.40329a66ba4e990f8463ecffc5dfdaa9. after waiting 1 ms 2024-11-26T14:31:47,376 INFO [RS:2;19945b63c2b4:37467 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-26T14:31:47,376 INFO [RS:2;19945b63c2b4:37467 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-26T14:31:47,376 DEBUG [RS_CLOSE_REGION-regionserver/19945b63c2b4:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsAfterRegionMovedWithMultiCF,,1732631484097.40329a66ba4e990f8463ecffc5dfdaa9. 2024-11-26T14:31:47,376 INFO [RS:2;19945b63c2b4:37467 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-11-26T14:31:47,376 INFO [RS:2;19945b63c2b4:37467 {}] regionserver.HRegionServer(1321): Waiting on 2 regions to close 2024-11-26T14:31:47,376 DEBUG [RS:2;19945b63c2b4:37467 {}] regionserver.HRegionServer(1325): Online Regions={1588230740=hbase:meta,,1.1588230740, 40329a66ba4e990f8463ecffc5dfdaa9=testReplayEditsAfterRegionMovedWithMultiCF,,1732631484097.40329a66ba4e990f8463ecffc5dfdaa9.} 2024-11-26T14:31:47,376 DEBUG [RS:2;19945b63c2b4:37467 {}] regionserver.HRegionServer(1351): Waiting on 1588230740, 40329a66ba4e990f8463ecffc5dfdaa9 2024-11-26T14:31:47,376 DEBUG [RS_CLOSE_META-regionserver/19945b63c2b4:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-26T14:31:47,376 INFO [RS_CLOSE_META-regionserver/19945b63c2b4:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-26T14:31:47,377 DEBUG [RS_CLOSE_META-regionserver/19945b63c2b4:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-26T14:31:47,377 DEBUG [RS_CLOSE_META-regionserver/19945b63c2b4:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-26T14:31:47,377 DEBUG [RS_CLOSE_META-regionserver/19945b63c2b4:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-26T14:31:47,377 INFO [RS_CLOSE_META-regionserver/19945b63c2b4:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=6.86 KB heapSize=11.45 KB 2024-11-26T14:31:47,381 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38147 is added to blk_1073741833_1009 (size=2054) 2024-11-26T14:31:47,381 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36307 is added to blk_1073741833_1009 (size=2054) 2024-11-26T14:31:47,381 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36277 is added to blk_1073741833_1009 (size=2054) 2024-11-26T14:31:47,384 DEBUG [RS:1;19945b63c2b4:40179 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/2de93217-0593-bca0-5c5f-879b6362c76f/oldWALs 2024-11-26T14:31:47,384 INFO [RS:1;19945b63c2b4:40179 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 19945b63c2b4%2C40179%2C1732631469496:(num 1732631471133) 2024-11-26T14:31:47,384 DEBUG [RS_CLOSE_REGION-regionserver/19945b63c2b4:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44999/user/jenkins/test-data/2de93217-0593-bca0-5c5f-879b6362c76f/data/default/testReplayEditsAfterRegionMovedWithMultiCF/40329a66ba4e990f8463ecffc5dfdaa9/recovered.edits/20.seqid, newMaxSeqId=20, maxSeqId=17 2024-11-26T14:31:47,384 DEBUG [RS:1;19945b63c2b4:40179 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T14:31:47,384 INFO [RS:1;19945b63c2b4:40179 {}] regionserver.LeaseManager(133): Closed leases 2024-11-26T14:31:47,384 INFO [RS:1;19945b63c2b4:40179 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-26T14:31:47,384 INFO [RS:1;19945b63c2b4:40179 {}] hbase.ChoreService(370): Chore service for: regionserver/19945b63c2b4:0 had [ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-11-26T14:31:47,384 INFO [RS:1;19945b63c2b4:40179 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-26T14:31:47,384 INFO [regionserver/19945b63c2b4:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-26T14:31:47,384 INFO [RS:1;19945b63c2b4:40179 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-26T14:31:47,384 INFO [RS:1;19945b63c2b4:40179 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-26T14:31:47,384 INFO [RS:1;19945b63c2b4:40179 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-26T14:31:47,385 INFO [RS:1;19945b63c2b4:40179 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.3:40179 2024-11-26T14:31:47,385 INFO [RS_CLOSE_REGION-regionserver/19945b63c2b4:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testReplayEditsAfterRegionMovedWithMultiCF,,1732631484097.40329a66ba4e990f8463ecffc5dfdaa9. 2024-11-26T14:31:47,385 DEBUG [RS_CLOSE_REGION-regionserver/19945b63c2b4:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 40329a66ba4e990f8463ecffc5dfdaa9: Waiting for close lock at 1732631507375Running coprocessor pre-close hooks at 1732631507375Disabling compacts and flushes for region at 1732631507375Disabling writes for close at 1732631507376 (+1 ms)Writing region close event to WAL at 1732631507379 (+3 ms)Running coprocessor post-close hooks at 1732631507385 (+6 ms)Closed at 1732631507385 2024-11-26T14:31:47,385 DEBUG [RS_CLOSE_REGION-regionserver/19945b63c2b4:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testReplayEditsAfterRegionMovedWithMultiCF,,1732631484097.40329a66ba4e990f8463ecffc5dfdaa9. 2024-11-26T14:31:47,395 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40179-0x101778e13450002, quorum=127.0.0.1:49403, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/19945b63c2b4,40179,1732631469496 2024-11-26T14:31:47,395 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43577-0x101778e13450000, quorum=127.0.0.1:49403, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-26T14:31:47,396 INFO [RS:1;19945b63c2b4:40179 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-26T14:31:47,399 DEBUG [RS_CLOSE_META-regionserver/19945b63c2b4:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44999/user/jenkins/test-data/2de93217-0593-bca0-5c5f-879b6362c76f/data/hbase/meta/1588230740/.tmp/info/3126c05150844591903e7b513145d789 is 205, key is testReplayEditsAfterRegionMovedWithMultiCF,,1732631484097.40329a66ba4e990f8463ecffc5dfdaa9./info:regioninfo/1732631487359/Put/seqid=0 2024-11-26T14:31:47,404 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36307 is added to blk_1073741938_1118 (size=8243) 2024-11-26T14:31:47,404 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36277 is added to blk_1073741938_1118 (size=8243) 2024-11-26T14:31:47,404 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38147 is added to blk_1073741938_1118 (size=8243) 2024-11-26T14:31:47,405 INFO [RS_CLOSE_META-regionserver/19945b63c2b4:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=6.65 KB at sequenceid=21 (bloomFilter=true), to=hdfs://localhost:44999/user/jenkins/test-data/2de93217-0593-bca0-5c5f-879b6362c76f/data/hbase/meta/1588230740/.tmp/info/3126c05150844591903e7b513145d789 2024-11-26T14:31:47,406 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [19945b63c2b4,40179,1732631469496] 2024-11-26T14:31:47,416 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/19945b63c2b4,40179,1732631469496 already deleted, retry=false 2024-11-26T14:31:47,417 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; 19945b63c2b4,40179,1732631469496 expired; onlineServers=1 2024-11-26T14:31:47,418 INFO [regionserver/19945b63c2b4:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-26T14:31:47,418 INFO [regionserver/19945b63c2b4:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-26T14:31:47,423 DEBUG [RS_CLOSE_META-regionserver/19945b63c2b4:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44999/user/jenkins/test-data/2de93217-0593-bca0-5c5f-879b6362c76f/data/hbase/meta/1588230740/.tmp/ns/4b84d43b041b47d799f33df04a9e0b25 is 43, key is default/ns:d/1732631471618/Put/seqid=0 2024-11-26T14:31:47,428 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36307 is added to blk_1073741939_1119 (size=5153) 2024-11-26T14:31:47,428 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36277 is added to blk_1073741939_1119 (size=5153) 2024-11-26T14:31:47,429 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38147 is added to blk_1073741939_1119 (size=5153) 2024-11-26T14:31:47,429 INFO [RS_CLOSE_META-regionserver/19945b63c2b4:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=74 B at sequenceid=21 (bloomFilter=true), to=hdfs://localhost:44999/user/jenkins/test-data/2de93217-0593-bca0-5c5f-879b6362c76f/data/hbase/meta/1588230740/.tmp/ns/4b84d43b041b47d799f33df04a9e0b25 2024-11-26T14:31:47,446 DEBUG [RS_CLOSE_META-regionserver/19945b63c2b4:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44999/user/jenkins/test-data/2de93217-0593-bca0-5c5f-879b6362c76f/data/hbase/meta/1588230740/.tmp/table/9408bc5e49c648c1a6974162fab5ede5 is 78, key is testReplayEditsAfterRegionMovedWithMultiCF/table:state/1732631484521/Put/seqid=0 2024-11-26T14:31:47,452 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38147 is added to blk_1073741940_1120 (size=5431) 2024-11-26T14:31:47,452 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36307 is added to blk_1073741940_1120 (size=5431) 2024-11-26T14:31:47,452 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36277 is added to blk_1073741940_1120 (size=5431) 2024-11-26T14:31:47,453 INFO [RS_CLOSE_META-regionserver/19945b63c2b4:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=148 B at sequenceid=21 (bloomFilter=true), to=hdfs://localhost:44999/user/jenkins/test-data/2de93217-0593-bca0-5c5f-879b6362c76f/data/hbase/meta/1588230740/.tmp/table/9408bc5e49c648c1a6974162fab5ede5 2024-11-26T14:31:47,458 DEBUG [RS_CLOSE_META-regionserver/19945b63c2b4:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44999/user/jenkins/test-data/2de93217-0593-bca0-5c5f-879b6362c76f/data/hbase/meta/1588230740/.tmp/info/3126c05150844591903e7b513145d789 as hdfs://localhost:44999/user/jenkins/test-data/2de93217-0593-bca0-5c5f-879b6362c76f/data/hbase/meta/1588230740/info/3126c05150844591903e7b513145d789 2024-11-26T14:31:47,463 INFO [RS_CLOSE_META-regionserver/19945b63c2b4:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44999/user/jenkins/test-data/2de93217-0593-bca0-5c5f-879b6362c76f/data/hbase/meta/1588230740/info/3126c05150844591903e7b513145d789, entries=18, sequenceid=21, filesize=8.0 K 2024-11-26T14:31:47,464 DEBUG [RS_CLOSE_META-regionserver/19945b63c2b4:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44999/user/jenkins/test-data/2de93217-0593-bca0-5c5f-879b6362c76f/data/hbase/meta/1588230740/.tmp/ns/4b84d43b041b47d799f33df04a9e0b25 as hdfs://localhost:44999/user/jenkins/test-data/2de93217-0593-bca0-5c5f-879b6362c76f/data/hbase/meta/1588230740/ns/4b84d43b041b47d799f33df04a9e0b25 2024-11-26T14:31:47,468 INFO [RS_CLOSE_META-regionserver/19945b63c2b4:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44999/user/jenkins/test-data/2de93217-0593-bca0-5c5f-879b6362c76f/data/hbase/meta/1588230740/ns/4b84d43b041b47d799f33df04a9e0b25, entries=2, sequenceid=21, filesize=5.0 K 2024-11-26T14:31:47,469 DEBUG [RS_CLOSE_META-regionserver/19945b63c2b4:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44999/user/jenkins/test-data/2de93217-0593-bca0-5c5f-879b6362c76f/data/hbase/meta/1588230740/.tmp/table/9408bc5e49c648c1a6974162fab5ede5 as hdfs://localhost:44999/user/jenkins/test-data/2de93217-0593-bca0-5c5f-879b6362c76f/data/hbase/meta/1588230740/table/9408bc5e49c648c1a6974162fab5ede5 2024-11-26T14:31:47,474 INFO [RS_CLOSE_META-regionserver/19945b63c2b4:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44999/user/jenkins/test-data/2de93217-0593-bca0-5c5f-879b6362c76f/data/hbase/meta/1588230740/table/9408bc5e49c648c1a6974162fab5ede5, entries=2, sequenceid=21, filesize=5.3 K 2024-11-26T14:31:47,475 INFO [RS_CLOSE_META-regionserver/19945b63c2b4:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~6.86 KB/7029, heapSize ~11.16 KB/11424, currentSize=0 B/0 for 1588230740 in 97ms, sequenceid=21, compaction requested=false 2024-11-26T14:31:47,478 DEBUG [RS_CLOSE_META-regionserver/19945b63c2b4:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44999/user/jenkins/test-data/2de93217-0593-bca0-5c5f-879b6362c76f/data/hbase/meta/1588230740/recovered.edits/24.seqid, newMaxSeqId=24, maxSeqId=1 2024-11-26T14:31:47,479 DEBUG [RS_CLOSE_META-regionserver/19945b63c2b4:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-26T14:31:47,479 INFO [RS_CLOSE_META-regionserver/19945b63c2b4:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-26T14:31:47,479 DEBUG [RS_CLOSE_META-regionserver/19945b63c2b4:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1732631507376Running coprocessor pre-close hooks at 1732631507376Disabling compacts and flushes for region at 1732631507376Disabling writes for close at 1732631507377 (+1 ms)Obtaining lock to block concurrent updates at 1732631507377Preparing flush snapshotting stores in 1588230740 at 1732631507377Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=7029, getHeapSize=11664, getOffHeapSize=0, getCellsCount=48 at 1732631507378 (+1 ms)Flushing stores of hbase:meta,,1.1588230740 at 1732631507378Flushing 1588230740/info: creating writer at 1732631507379 (+1 ms)Flushing 1588230740/info: appending metadata at 1732631507398 (+19 ms)Flushing 1588230740/info: closing flushed file at 1732631507398Flushing 1588230740/ns: creating writer at 1732631507410 (+12 ms)Flushing 1588230740/ns: appending metadata at 1732631507422 (+12 ms)Flushing 1588230740/ns: closing flushed file at 1732631507422Flushing 1588230740/table: creating writer at 1732631507433 (+11 ms)Flushing 1588230740/table: appending metadata at 1732631507446 (+13 ms)Flushing 1588230740/table: closing flushed file at 1732631507446Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@251df1b: reopening flushed file at 1732631507457 (+11 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@6075fece: reopening flushed file at 1732631507463 (+6 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@5654b1ac: reopening flushed file at 1732631507469 (+6 ms)Finished flush of dataSize ~6.86 KB/7029, heapSize ~11.16 KB/11424, currentSize=0 B/0 for 1588230740 in 97ms, sequenceid=21, compaction requested=false at 1732631507475 (+6 ms)Writing region close event to WAL at 1732631507476 (+1 ms)Running coprocessor post-close hooks at 1732631507479 (+3 ms)Closed at 1732631507479 2024-11-26T14:31:47,479 DEBUG [RS_CLOSE_META-regionserver/19945b63c2b4:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-11-26T14:31:47,506 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40179-0x101778e13450002, quorum=127.0.0.1:49403, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-26T14:31:47,506 INFO [RS:1;19945b63c2b4:40179 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-26T14:31:47,506 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40179-0x101778e13450002, quorum=127.0.0.1:49403, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-26T14:31:47,506 INFO [RS:1;19945b63c2b4:40179 {}] regionserver.HRegionServer(1031): Exiting; stopping=19945b63c2b4,40179,1732631469496; zookeeper connection closed. 2024-11-26T14:31:47,507 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@2510cd15 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@2510cd15 2024-11-26T14:31:47,576 INFO [RS:2;19945b63c2b4:37467 {}] regionserver.HRegionServer(976): stopping server 19945b63c2b4,37467,1732631469538; all regions closed. 2024-11-26T14:31:47,582 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36277 is added to blk_1073741836_1012 (size=3783) 2024-11-26T14:31:47,582 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38147 is added to blk_1073741836_1012 (size=3783) 2024-11-26T14:31:47,582 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36307 is added to blk_1073741836_1012 (size=3783) 2024-11-26T14:31:47,586 DEBUG [RS:2;19945b63c2b4:37467 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/2de93217-0593-bca0-5c5f-879b6362c76f/oldWALs 2024-11-26T14:31:47,587 INFO [RS:2;19945b63c2b4:37467 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 19945b63c2b4%2C37467%2C1732631469538.meta:.meta(num 1732631471457) 2024-11-26T14:31:47,590 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38147 is added to blk_1073741834_1010 (size=841) 2024-11-26T14:31:47,590 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36277 is added to blk_1073741834_1010 (size=841) 2024-11-26T14:31:47,590 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36307 is added to blk_1073741834_1010 (size=841) 2024-11-26T14:31:47,592 DEBUG [RS:2;19945b63c2b4:37467 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/2de93217-0593-bca0-5c5f-879b6362c76f/oldWALs 2024-11-26T14:31:47,592 INFO [RS:2;19945b63c2b4:37467 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 19945b63c2b4%2C37467%2C1732631469538:(num 1732631471132) 2024-11-26T14:31:47,592 DEBUG [RS:2;19945b63c2b4:37467 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T14:31:47,592 INFO [RS:2;19945b63c2b4:37467 {}] regionserver.LeaseManager(133): Closed leases 2024-11-26T14:31:47,593 INFO [RS:2;19945b63c2b4:37467 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-26T14:31:47,593 INFO [RS:2;19945b63c2b4:37467 {}] hbase.ChoreService(370): Chore service for: regionserver/19945b63c2b4:0 had [ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS] on shutdown 2024-11-26T14:31:47,593 INFO [RS:2;19945b63c2b4:37467 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-26T14:31:47,593 INFO [regionserver/19945b63c2b4:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-26T14:31:47,593 INFO [RS:2;19945b63c2b4:37467 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.3:37467 2024-11-26T14:31:47,606 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37467-0x101778e13450003, quorum=127.0.0.1:49403, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/19945b63c2b4,37467,1732631469538 2024-11-26T14:31:47,606 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43577-0x101778e13450000, quorum=127.0.0.1:49403, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-26T14:31:47,606 INFO [RS:2;19945b63c2b4:37467 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-26T14:31:47,617 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [19945b63c2b4,37467,1732631469538] 2024-11-26T14:31:47,627 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/19945b63c2b4,37467,1732631469538 already deleted, retry=false 2024-11-26T14:31:47,627 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; 19945b63c2b4,37467,1732631469538 expired; onlineServers=0 2024-11-26T14:31:47,628 INFO [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master '19945b63c2b4,43577,1732631468672' ***** 2024-11-26T14:31:47,628 INFO [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0 2024-11-26T14:31:47,628 INFO [M:0;19945b63c2b4:43577 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-26T14:31:47,628 INFO [M:0;19945b63c2b4:43577 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-26T14:31:47,628 DEBUG [M:0;19945b63c2b4:43577 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2024-11-26T14:31:47,629 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2024-11-26T14:31:47,629 DEBUG [M:0;19945b63c2b4:43577 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2024-11-26T14:31:47,629 DEBUG [master/19945b63c2b4:0:becomeActiveMaster-HFileCleaner.large.0-1732631470712 {}] cleaner.HFileCleaner(306): Exit Thread[master/19945b63c2b4:0:becomeActiveMaster-HFileCleaner.large.0-1732631470712,5,FailOnTimeoutGroup] 2024-11-26T14:31:47,629 DEBUG [master/19945b63c2b4:0:becomeActiveMaster-HFileCleaner.small.0-1732631470713 {}] cleaner.HFileCleaner(306): Exit Thread[master/19945b63c2b4:0:becomeActiveMaster-HFileCleaner.small.0-1732631470713,5,FailOnTimeoutGroup] 2024-11-26T14:31:47,629 INFO [M:0;19945b63c2b4:43577 {}] hbase.ChoreService(370): Chore service for: master/19945b63c2b4:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown 2024-11-26T14:31:47,630 INFO [M:0;19945b63c2b4:43577 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-26T14:31:47,630 DEBUG [M:0;19945b63c2b4:43577 {}] master.HMaster(1795): Stopping service threads 2024-11-26T14:31:47,630 INFO [M:0;19945b63c2b4:43577 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2024-11-26T14:31:47,630 INFO [M:0;19945b63c2b4:43577 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-26T14:31:47,632 INFO [M:0;19945b63c2b4:43577 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-26T14:31:47,632 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2024-11-26T14:31:47,638 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43577-0x101778e13450000, quorum=127.0.0.1:49403, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2024-11-26T14:31:47,638 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43577-0x101778e13450000, quorum=127.0.0.1:49403, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-26T14:31:47,638 DEBUG [M:0;19945b63c2b4:43577 {}] zookeeper.ZKUtil(347): master:43577-0x101778e13450000, quorum=127.0.0.1:49403, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error) 2024-11-26T14:31:47,639 WARN [M:0;19945b63c2b4:43577 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null 2024-11-26T14:31:47,640 INFO [M:0;19945b63c2b4:43577 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: hdfs://localhost:44999/user/jenkins/test-data/2de93217-0593-bca0-5c5f-879b6362c76f/.lastflushedseqids 2024-11-26T14:31:47,654 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36277 is added to blk_1073741941_1121 (size=138) 2024-11-26T14:31:47,654 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36307 is added to blk_1073741941_1121 (size=138) 2024-11-26T14:31:47,654 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38147 is added to blk_1073741941_1121 (size=138) 2024-11-26T14:31:47,655 INFO [M:0;19945b63c2b4:43577 {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-26T14:31:47,655 INFO [M:0;19945b63c2b4:43577 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false 2024-11-26T14:31:47,655 DEBUG [M:0;19945b63c2b4:43577 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-26T14:31:47,655 INFO [M:0;19945b63c2b4:43577 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-26T14:31:47,655 DEBUG [M:0;19945b63c2b4:43577 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-26T14:31:47,655 DEBUG [M:0;19945b63c2b4:43577 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-26T14:31:47,655 DEBUG [M:0;19945b63c2b4:43577 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-26T14:31:47,655 INFO [M:0;19945b63c2b4:43577 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=68.36 KB heapSize=83.73 KB 2024-11-26T14:31:47,670 DEBUG [M:0;19945b63c2b4:43577 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44999/user/jenkins/test-data/2de93217-0593-bca0-5c5f-879b6362c76f/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/8255998aaf0a40688dd9512ab4772563 is 82, key is hbase:meta,,1/info:regioninfo/1732631471537/Put/seqid=0 2024-11-26T14:31:47,675 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36307 is added to blk_1073741942_1122 (size=5672) 2024-11-26T14:31:47,675 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38147 is added to blk_1073741942_1122 (size=5672) 2024-11-26T14:31:47,675 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36277 is added to blk_1073741942_1122 (size=5672) 2024-11-26T14:31:47,676 INFO [M:0;19945b63c2b4:43577 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=504 B at sequenceid=168 (bloomFilter=true), to=hdfs://localhost:44999/user/jenkins/test-data/2de93217-0593-bca0-5c5f-879b6362c76f/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/8255998aaf0a40688dd9512ab4772563 2024-11-26T14:31:47,695 DEBUG [M:0;19945b63c2b4:43577 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44999/user/jenkins/test-data/2de93217-0593-bca0-5c5f-879b6362c76f/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/2db8f2f05ff6435588f7c99b84fdcfe6 is 1077, key is \x00\x00\x00\x00\x00\x00\x00\x04/proc:d/1732631484528/Put/seqid=0 2024-11-26T14:31:47,707 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38147 is added to blk_1073741943_1123 (size=7756) 2024-11-26T14:31:47,707 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36307 is added to blk_1073741943_1123 (size=7756) 2024-11-26T14:31:47,707 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36277 is added to blk_1073741943_1123 (size=7756) 2024-11-26T14:31:47,708 INFO [M:0;19945b63c2b4:43577 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=67.62 KB at sequenceid=168 (bloomFilter=true), to=hdfs://localhost:44999/user/jenkins/test-data/2de93217-0593-bca0-5c5f-879b6362c76f/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/2db8f2f05ff6435588f7c99b84fdcfe6 2024-11-26T14:31:47,713 INFO [M:0;19945b63c2b4:43577 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 2db8f2f05ff6435588f7c99b84fdcfe6 2024-11-26T14:31:47,717 INFO [RS:2;19945b63c2b4:37467 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-26T14:31:47,717 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37467-0x101778e13450003, quorum=127.0.0.1:49403, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-26T14:31:47,717 INFO [RS:2;19945b63c2b4:37467 {}] regionserver.HRegionServer(1031): Exiting; stopping=19945b63c2b4,37467,1732631469538; zookeeper connection closed. 2024-11-26T14:31:47,717 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37467-0x101778e13450003, quorum=127.0.0.1:49403, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-26T14:31:47,717 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@352c59ee {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@352c59ee 2024-11-26T14:31:47,718 INFO [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 3 regionserver(s) complete 2024-11-26T14:31:47,733 DEBUG [M:0;19945b63c2b4:43577 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44999/user/jenkins/test-data/2de93217-0593-bca0-5c5f-879b6362c76f/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/03e50dc97a6d4403a238196499769d56 is 69, key is 19945b63c2b4,37467,1732631469538/rs:state/1732631470853/Put/seqid=0 2024-11-26T14:31:47,739 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36307 is added to blk_1073741944_1124 (size=5440) 2024-11-26T14:31:47,739 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38147 is added to blk_1073741944_1124 (size=5440) 2024-11-26T14:31:47,739 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36277 is added to blk_1073741944_1124 (size=5440) 2024-11-26T14:31:47,740 INFO [M:0;19945b63c2b4:43577 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=249 B at sequenceid=168 (bloomFilter=true), to=hdfs://localhost:44999/user/jenkins/test-data/2de93217-0593-bca0-5c5f-879b6362c76f/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/03e50dc97a6d4403a238196499769d56 2024-11-26T14:31:47,744 INFO [M:0;19945b63c2b4:43577 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 03e50dc97a6d4403a238196499769d56 2024-11-26T14:31:47,745 DEBUG [M:0;19945b63c2b4:43577 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44999/user/jenkins/test-data/2de93217-0593-bca0-5c5f-879b6362c76f/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/8255998aaf0a40688dd9512ab4772563 as hdfs://localhost:44999/user/jenkins/test-data/2de93217-0593-bca0-5c5f-879b6362c76f/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/8255998aaf0a40688dd9512ab4772563 2024-11-26T14:31:47,751 INFO [M:0;19945b63c2b4:43577 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44999/user/jenkins/test-data/2de93217-0593-bca0-5c5f-879b6362c76f/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/8255998aaf0a40688dd9512ab4772563, entries=8, sequenceid=168, filesize=5.5 K 2024-11-26T14:31:47,752 DEBUG [M:0;19945b63c2b4:43577 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44999/user/jenkins/test-data/2de93217-0593-bca0-5c5f-879b6362c76f/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/2db8f2f05ff6435588f7c99b84fdcfe6 as hdfs://localhost:44999/user/jenkins/test-data/2de93217-0593-bca0-5c5f-879b6362c76f/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/2db8f2f05ff6435588f7c99b84fdcfe6 2024-11-26T14:31:47,757 INFO [M:0;19945b63c2b4:43577 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 2db8f2f05ff6435588f7c99b84fdcfe6 2024-11-26T14:31:47,757 INFO [M:0;19945b63c2b4:43577 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44999/user/jenkins/test-data/2de93217-0593-bca0-5c5f-879b6362c76f/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/2db8f2f05ff6435588f7c99b84fdcfe6, entries=17, sequenceid=168, filesize=7.6 K 2024-11-26T14:31:47,758 DEBUG [M:0;19945b63c2b4:43577 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44999/user/jenkins/test-data/2de93217-0593-bca0-5c5f-879b6362c76f/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/03e50dc97a6d4403a238196499769d56 as hdfs://localhost:44999/user/jenkins/test-data/2de93217-0593-bca0-5c5f-879b6362c76f/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/03e50dc97a6d4403a238196499769d56 2024-11-26T14:31:47,763 INFO [M:0;19945b63c2b4:43577 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 03e50dc97a6d4403a238196499769d56 2024-11-26T14:31:47,763 INFO [M:0;19945b63c2b4:43577 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44999/user/jenkins/test-data/2de93217-0593-bca0-5c5f-879b6362c76f/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/03e50dc97a6d4403a238196499769d56, entries=3, sequenceid=168, filesize=5.3 K 2024-11-26T14:31:47,764 INFO [M:0;19945b63c2b4:43577 {}] regionserver.HRegion(3140): Finished flush of dataSize ~68.36 KB/69996, heapSize ~83.44 KB/85440, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 109ms, sequenceid=168, compaction requested=false 2024-11-26T14:31:47,766 INFO [M:0;19945b63c2b4:43577 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-26T14:31:47,766 DEBUG [M:0;19945b63c2b4:43577 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1732631507655Disabling compacts and flushes for region at 1732631507655Disabling writes for close at 1732631507655Obtaining lock to block concurrent updates at 1732631507655Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1732631507655Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=69996, getHeapSize=85680, getOffHeapSize=0, getCellsCount=195 at 1732631507655Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1732631507656 (+1 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1732631507656Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1732631507670 (+14 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1732631507670Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1732631507680 (+10 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1732631507694 (+14 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1732631507695 (+1 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1732631507713 (+18 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1732631507732 (+19 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1732631507732Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@66d56c1c: reopening flushed file at 1732631507744 (+12 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@201cb288: reopening flushed file at 1732631507751 (+7 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@7f511f0e: reopening flushed file at 1732631507757 (+6 ms)Finished flush of dataSize ~68.36 KB/69996, heapSize ~83.44 KB/85440, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 109ms, sequenceid=168, compaction requested=false at 1732631507764 (+7 ms)Writing region close event to WAL at 1732631507766 (+2 ms)Closed at 1732631507766 2024-11-26T14:31:47,768 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36307 is added to blk_1073741830_1006 (size=71349) 2024-11-26T14:31:47,768 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38147 is added to blk_1073741830_1006 (size=71349) 2024-11-26T14:31:47,768 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36277 is added to blk_1073741830_1006 (size=71349) 2024-11-26T14:31:47,771 INFO [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-26T14:31:47,771 INFO [M:0;19945b63c2b4:43577 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down. 2024-11-26T14:31:47,771 INFO [M:0;19945b63c2b4:43577 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.3:43577 2024-11-26T14:31:47,772 INFO [M:0;19945b63c2b4:43577 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-26T14:31:47,882 INFO [M:0;19945b63c2b4:43577 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-26T14:31:47,882 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43577-0x101778e13450000, quorum=127.0.0.1:49403, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-26T14:31:47,882 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43577-0x101778e13450000, quorum=127.0.0.1:49403, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-26T14:31:47,888 ERROR [Time-limited test {}] hdfs.DFSClient(665): Failed to close file: /hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1732631487529/wal.1732631487829 with renewLeaseKey: DEFAULT_16655 java.io.FileNotFoundException: File does not exist: /hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1732631487529/wal.1732631487829 (inode 16655) Holder DFSClient_NONMAPREDUCE_2043263076_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.ipc.RemoteException.instantiateException(RemoteException.java:121) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.RemoteException.unwrapRemoteException(RemoteException.java:110) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:653) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.close(FanOutOneBlockAsyncDFSOutput.java:619) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hdfs.DummyDFSOutputStream.close(DummyDFSOutputStream.java:52) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeAllFilesBeingWritten(DFSClient.java:662) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeOutputStreams(DFSClient.java:699) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.close(DistributedFileSystem.java:1528) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2194) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2174) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2167) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniDFSCluster(HBaseTestingUtil.java:761) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1021) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.ipc.RemoteException: File does not exist: /hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1732631487529/wal.1732631487829 (inode 16655) Holder DFSClient_NONMAPREDUCE_2043263076_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1584) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy43.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.lambda$complete$13(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.complete(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:437) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:170) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:162) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:100) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:366) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy44.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:646) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 25 more 2024-11-26T14:31:47,890 ERROR [Time-limited test {}] hdfs.DFSClient(665): Failed to close file: /hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1732631479646/wal.1732631479779 with renewLeaseKey: DEFAULT_16586 java.io.IOException: stream already broken at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.endBlock(FanOutOneBlockAsyncDFSOutput.java:566) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.close(FanOutOneBlockAsyncDFSOutput.java:615) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hdfs.DummyDFSOutputStream.close(DummyDFSOutputStream.java:52) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeAllFilesBeingWritten(DFSClient.java:662) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeOutputStreams(DFSClient.java:699) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.close(DistributedFileSystem.java:1528) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2194) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2174) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2167) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniDFSCluster(HBaseTestingUtil.java:761) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1021) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T14:31:47,890 ERROR [Time-limited test {}] hdfs.DFSClient(665): Failed to close file: /hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732631502656/wal.1732631502842 with renewLeaseKey: DEFAULT_16767 java.io.IOException: stream already broken at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.endBlock(FanOutOneBlockAsyncDFSOutput.java:566) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.close(FanOutOneBlockAsyncDFSOutput.java:615) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hdfs.DummyDFSOutputStream.close(DummyDFSOutputStream.java:52) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeAllFilesBeingWritten(DFSClient.java:662) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeOutputStreams(DFSClient.java:699) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.close(DistributedFileSystem.java:1528) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2194) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2174) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2167) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniDFSCluster(HBaseTestingUtil.java:761) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1021) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T14:31:47,893 ERROR [Time-limited test {}] hdfs.DFSClient(665): Failed to close file: /hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1732631488070/wal.1732631497325 with renewLeaseKey: DEFAULT_16678 java.io.FileNotFoundException: File does not exist: /hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1732631488070/wal.1732631497325 (inode 16678) Holder DFSClient_NONMAPREDUCE_2043263076_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.ipc.RemoteException.instantiateException(RemoteException.java:121) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.RemoteException.unwrapRemoteException(RemoteException.java:110) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:653) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.close(FanOutOneBlockAsyncDFSOutput.java:619) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hdfs.DummyDFSOutputStream.close(DummyDFSOutputStream.java:52) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeAllFilesBeingWritten(DFSClient.java:662) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeOutputStreams(DFSClient.java:699) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.close(DistributedFileSystem.java:1528) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2194) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2174) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2167) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniDFSCluster(HBaseTestingUtil.java:761) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1021) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.ipc.RemoteException: File does not exist: /hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1732631488070/wal.1732631497325 (inode 16678) Holder DFSClient_NONMAPREDUCE_2043263076_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1584) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy43.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.lambda$complete$13(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.complete(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:437) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:170) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:162) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:100) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:366) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy44.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:646) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 25 more 2024-11-26T14:31:47,894 ERROR [Time-limited test {}] hdfs.DFSClient(665): Failed to close file: /hbase/WALs/testreplayeditswrittenintowal-manual,16010,1732631472647/wal.1732631472718 with renewLeaseKey: DEFAULT_16506 java.io.IOException: stream already broken at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.endBlock(FanOutOneBlockAsyncDFSOutput.java:566) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.close(FanOutOneBlockAsyncDFSOutput.java:615) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hdfs.DummyDFSOutputStream.close(DummyDFSOutputStream.java:52) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeAllFilesBeingWritten(DFSClient.java:662) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeOutputStreams(DFSClient.java:699) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.close(DistributedFileSystem.java:1528) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2194) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2174) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2167) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniDFSCluster(HBaseTestingUtil.java:761) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1021) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T14:31:47,896 ERROR [Time-limited test {}] hdfs.DFSClient(665): Failed to close file: /hbase/WALs/testdatalosswheninputerror-manual,16010,1732631497524/wal.1732631498022 with renewLeaseKey: DEFAULT_16704 java.io.FileNotFoundException: File does not exist: /hbase/WALs/testdatalosswheninputerror-manual,16010,1732631497524/wal.1732631498022 (inode 16704) Holder DFSClient_NONMAPREDUCE_2043263076_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.ipc.RemoteException.instantiateException(RemoteException.java:121) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.RemoteException.unwrapRemoteException(RemoteException.java:110) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:653) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.close(FanOutOneBlockAsyncDFSOutput.java:619) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hdfs.DummyDFSOutputStream.close(DummyDFSOutputStream.java:52) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeAllFilesBeingWritten(DFSClient.java:662) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeOutputStreams(DFSClient.java:699) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.close(DistributedFileSystem.java:1528) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2194) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2174) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2167) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniDFSCluster(HBaseTestingUtil.java:761) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1021) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.ipc.RemoteException: File does not exist: /hbase/WALs/testdatalosswheninputerror-manual,16010,1732631497524/wal.1732631498022 (inode 16704) Holder DFSClient_NONMAPREDUCE_2043263076_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1584) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy43.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.lambda$complete$13(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.complete(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:437) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:170) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:162) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:100) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:366) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy44.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:646) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 25 more 2024-11-26T14:31:47,897 ERROR [Time-limited test {}] hdfs.DFSClient(665): Failed to close file: /hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1732631498166/wal.1732631498235 with renewLeaseKey: DEFAULT_16726 java.io.IOException: stream already broken at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.endBlock(FanOutOneBlockAsyncDFSOutput.java:566) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.close(FanOutOneBlockAsyncDFSOutput.java:615) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hdfs.DummyDFSOutputStream.close(DummyDFSOutputStream.java:52) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeAllFilesBeingWritten(DFSClient.java:662) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeOutputStreams(DFSClient.java:699) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.close(DistributedFileSystem.java:1528) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2194) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2174) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2167) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniDFSCluster(HBaseTestingUtil.java:761) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1021) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T14:31:47,899 ERROR [Time-limited test {}] hdfs.DFSClient(665): Failed to close file: /hbase/WALs/testnameconflictwhensplit1-manual,16010,1732631472325/wal.1732631472505 with renewLeaseKey: DEFAULT_16485 java.io.FileNotFoundException: File does not exist: /hbase/WALs/testnameconflictwhensplit1-manual,16010,1732631472325/wal.1732631472505 (inode 16485) Holder DFSClient_NONMAPREDUCE_2043263076_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.ipc.RemoteException.instantiateException(RemoteException.java:121) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.RemoteException.unwrapRemoteException(RemoteException.java:110) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:653) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.close(FanOutOneBlockAsyncDFSOutput.java:619) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hdfs.DummyDFSOutputStream.close(DummyDFSOutputStream.java:52) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeAllFilesBeingWritten(DFSClient.java:662) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeOutputStreams(DFSClient.java:699) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.close(DistributedFileSystem.java:1528) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2194) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2174) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2167) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniDFSCluster(HBaseTestingUtil.java:761) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1021) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.ipc.RemoteException: File does not exist: /hbase/WALs/testnameconflictwhensplit1-manual,16010,1732631472325/wal.1732631472505 (inode 16485) Holder DFSClient_NONMAPREDUCE_2043263076_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1584) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy43.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.lambda$complete$13(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.complete(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:437) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:170) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:162) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:100) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:366) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy44.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:646) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 25 more 2024-11-26T14:31:47,901 ERROR [Time-limited test {}] hdfs.DFSClient(665): Failed to close file: /hbase/WALs/testnameconflictwhensplit0-manual,16010,1732631471861/wal.1732631472089 with renewLeaseKey: DEFAULT_16462 java.io.FileNotFoundException: File does not exist: /hbase/WALs/testnameconflictwhensplit0-manual,16010,1732631471861/wal.1732631472089 (inode 16462) Holder DFSClient_NONMAPREDUCE_2043263076_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.ipc.RemoteException.instantiateException(RemoteException.java:121) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.RemoteException.unwrapRemoteException(RemoteException.java:110) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:653) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.close(FanOutOneBlockAsyncDFSOutput.java:619) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hdfs.DummyDFSOutputStream.close(DummyDFSOutputStream.java:52) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeAllFilesBeingWritten(DFSClient.java:662) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeOutputStreams(DFSClient.java:699) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.close(DistributedFileSystem.java:1528) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2194) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2174) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2167) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniDFSCluster(HBaseTestingUtil.java:761) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1021) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.ipc.RemoteException: File does not exist: /hbase/WALs/testnameconflictwhensplit0-manual,16010,1732631471861/wal.1732631472089 (inode 16462) Holder DFSClient_NONMAPREDUCE_2043263076_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1584) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy43.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.lambda$complete$13(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.complete(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:437) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:170) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:162) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:100) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:366) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy44.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:646) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 25 more 2024-11-26T14:31:47,937 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@4290616c{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-26T14:31:47,940 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@5e25d2b6{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-26T14:31:47,940 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-26T14:31:47,940 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@6853f5bc{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-26T14:31:47,940 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@1bf0fdb4{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/175d8e9e-4f50-9d71-fd80-148275c72391/hadoop.log.dir/,STOPPED} 2024-11-26T14:31:47,943 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-11-26T14:31:47,943 WARN [BP-1347678837-172.17.0.3-1732631464290 heartbeating to localhost/127.0.0.1:44999 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-11-26T14:31:47,944 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-11-26T14:31:47,944 WARN [BP-1347678837-172.17.0.3-1732631464290 heartbeating to localhost/127.0.0.1:44999 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-1347678837-172.17.0.3-1732631464290 (Datanode Uuid d2854781-abc3-4956-9d56-42b6dfcc9e29) service to localhost/127.0.0.1:44999 2024-11-26T14:31:47,945 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/175d8e9e-4f50-9d71-fd80-148275c72391/cluster_49e351ba-434f-d14a-4712-fb0fcff718bc/data/data5/current/BP-1347678837-172.17.0.3-1732631464290 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-26T14:31:47,945 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/175d8e9e-4f50-9d71-fd80-148275c72391/cluster_49e351ba-434f-d14a-4712-fb0fcff718bc/data/data6/current/BP-1347678837-172.17.0.3-1732631464290 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-26T14:31:47,946 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-11-26T14:31:47,948 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@28e1ba78{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-26T14:31:47,949 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@702c0733{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-26T14:31:47,949 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-26T14:31:47,949 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@3264f7d{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-26T14:31:47,949 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@e971547{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/175d8e9e-4f50-9d71-fd80-148275c72391/hadoop.log.dir/,STOPPED} 2024-11-26T14:31:47,950 WARN [BP-1347678837-172.17.0.3-1732631464290 heartbeating to localhost/127.0.0.1:44999 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-11-26T14:31:47,950 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-11-26T14:31:47,950 WARN [BP-1347678837-172.17.0.3-1732631464290 heartbeating to localhost/127.0.0.1:44999 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-1347678837-172.17.0.3-1732631464290 (Datanode Uuid cf2dc361-1427-41a9-835e-da08867ae736) service to localhost/127.0.0.1:44999 2024-11-26T14:31:47,950 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-11-26T14:31:47,951 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/175d8e9e-4f50-9d71-fd80-148275c72391/cluster_49e351ba-434f-d14a-4712-fb0fcff718bc/data/data3/current/BP-1347678837-172.17.0.3-1732631464290 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-26T14:31:47,951 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/175d8e9e-4f50-9d71-fd80-148275c72391/cluster_49e351ba-434f-d14a-4712-fb0fcff718bc/data/data4/current/BP-1347678837-172.17.0.3-1732631464290 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-26T14:31:47,951 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-11-26T14:31:47,953 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@4dc44ec6{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-26T14:31:47,954 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@17f6e478{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-26T14:31:47,954 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-26T14:31:47,954 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@3d33512e{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-26T14:31:47,954 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@4b895d95{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/175d8e9e-4f50-9d71-fd80-148275c72391/hadoop.log.dir/,STOPPED} 2024-11-26T14:31:47,955 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-11-26T14:31:47,956 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-11-26T14:31:47,956 WARN [BP-1347678837-172.17.0.3-1732631464290 heartbeating to localhost/127.0.0.1:44999 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-11-26T14:31:47,956 WARN [BP-1347678837-172.17.0.3-1732631464290 heartbeating to localhost/127.0.0.1:44999 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-1347678837-172.17.0.3-1732631464290 (Datanode Uuid dc8cdb2d-b696-4216-9a09-4a5ee6dadb62) service to localhost/127.0.0.1:44999 2024-11-26T14:31:47,957 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/175d8e9e-4f50-9d71-fd80-148275c72391/cluster_49e351ba-434f-d14a-4712-fb0fcff718bc/data/data1/current/BP-1347678837-172.17.0.3-1732631464290 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-26T14:31:47,957 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/175d8e9e-4f50-9d71-fd80-148275c72391/cluster_49e351ba-434f-d14a-4712-fb0fcff718bc/data/data2/current/BP-1347678837-172.17.0.3-1732631464290 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-26T14:31:47,957 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-11-26T14:31:47,964 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@1843526c{hdfs,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-11-26T14:31:47,965 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@776c4d96{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-26T14:31:47,965 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-26T14:31:47,966 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@70596a48{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-26T14:31:47,966 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@6630be9c{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/175d8e9e-4f50-9d71-fd80-148275c72391/hadoop.log.dir/,STOPPED} 2024-11-26T14:31:47,977 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers 2024-11-26T14:31:48,025 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down