2024-11-24 07:35:07,095 main DEBUG Apache Log4j Core 2.17.2 initializing configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@20b2475a 2024-11-24 07:35:07,108 main DEBUG Took 0.011451 seconds to load 1 plugins from package org.apache.hadoop.hbase.logging 2024-11-24 07:35:07,109 main DEBUG PluginManager 'Core' found 129 plugins 2024-11-24 07:35:07,109 main DEBUG PluginManager 'Level' found 0 plugins 2024-11-24 07:35:07,110 main DEBUG PluginManager 'Lookup' found 16 plugins 2024-11-24 07:35:07,112 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-24 07:35:07,119 main DEBUG PluginManager 'TypeConverter' found 26 plugins 2024-11-24 07:35:07,134 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.util.MBeans", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-24 07:35:07,135 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-24 07:35:07,136 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.logging.TestJul2Slf4j", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-24 07:35:07,137 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-24 07:35:07,137 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.zookeeper", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-24 07:35:07,138 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-24 07:35:07,139 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSinkAdapter", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-24 07:35:07,139 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-24 07:35:07,140 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSystemImpl", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-24 07:35:07,140 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-24 07:35:07,141 main DEBUG LoggerConfig$Builder(additivity="false", level="WARN", levelAndRefs="null", name="org.apache.directory", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-24 07:35:07,142 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-24 07:35:07,142 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.ipc.FailedServers", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-24 07:35:07,143 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-24 07:35:07,143 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsConfig", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-24 07:35:07,144 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-24 07:35:07,144 main DEBUG LoggerConfig$Builder(additivity="null", level="INFO", levelAndRefs="null", name="org.apache.hadoop.hbase.ScheduledChore", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-24 07:35:07,145 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-24 07:35:07,145 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.regionserver.RSRpcServices", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-24 07:35:07,146 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-24 07:35:07,146 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-24 07:35:07,147 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-24 07:35:07,147 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-24 07:35:07,148 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-24 07:35:07,148 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hbase.thirdparty.io.netty.channel", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-24 07:35:07,148 main DEBUG Building Plugin[name=root, class=org.apache.logging.log4j.core.config.LoggerConfig$RootLogger]. 2024-11-24 07:35:07,150 main DEBUG LoggerConfig$RootLogger$Builder(additivity="null", level="null", levelAndRefs="INFO,Console", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-24 07:35:07,152 main DEBUG Building Plugin[name=loggers, class=org.apache.logging.log4j.core.config.LoggersPlugin]. 2024-11-24 07:35:07,154 main DEBUG createLoggers(={org.apache.hadoop.metrics2.util.MBeans, org.apache.hadoop.hbase.logging.TestJul2Slf4j, org.apache.zookeeper, org.apache.hadoop.metrics2.impl.MetricsSinkAdapter, org.apache.hadoop.metrics2.impl.MetricsSystemImpl, org.apache.directory, org.apache.hadoop.hbase.ipc.FailedServers, org.apache.hadoop.metrics2.impl.MetricsConfig, org.apache.hadoop.hbase.ScheduledChore, org.apache.hadoop.hbase.regionserver.RSRpcServices, org.apache.hadoop, org.apache.hadoop.hbase, org.apache.hbase.thirdparty.io.netty.channel, root}) 2024-11-24 07:35:07,154 main DEBUG Building Plugin[name=layout, class=org.apache.logging.log4j.core.layout.PatternLayout]. 2024-11-24 07:35:07,155 main DEBUG PatternLayout$Builder(pattern="%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n", PatternSelector=null, Configuration(PropertiesConfig), Replace=null, charset="null", alwaysWriteExceptions="null", disableAnsi="null", noConsoleNoAnsi="null", header="null", footer="null") 2024-11-24 07:35:07,156 main DEBUG PluginManager 'Converter' found 47 plugins 2024-11-24 07:35:07,163 main DEBUG Building Plugin[name=appender, class=org.apache.hadoop.hbase.logging.HBaseTestAppender]. 2024-11-24 07:35:07,166 main DEBUG HBaseTestAppender$Builder(target="SYSTEM_ERR", maxSize="1G", bufferedIo="null", bufferSize="null", immediateFlush="null", ignoreExceptions="null", PatternLayout(%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n), name="Console", Configuration(PropertiesConfig), Filter=null, ={}) 2024-11-24 07:35:07,167 main DEBUG Starting HBaseTestOutputStreamManager SYSTEM_ERR 2024-11-24 07:35:07,168 main DEBUG Building Plugin[name=appenders, class=org.apache.logging.log4j.core.config.AppendersPlugin]. 2024-11-24 07:35:07,168 main DEBUG createAppenders(={Console}) 2024-11-24 07:35:07,169 main DEBUG Configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@20b2475a initialized 2024-11-24 07:35:07,169 main DEBUG Starting configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@20b2475a 2024-11-24 07:35:07,169 main DEBUG Started configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@20b2475a OK. 2024-11-24 07:35:07,170 main DEBUG Shutting down OutputStreamManager SYSTEM_OUT.false.false-1 2024-11-24 07:35:07,170 main DEBUG OutputStream closed 2024-11-24 07:35:07,170 main DEBUG Shut down OutputStreamManager SYSTEM_OUT.false.false-1, all resources released: true 2024-11-24 07:35:07,170 main DEBUG Appender DefaultConsole-1 stopped with status true 2024-11-24 07:35:07,171 main DEBUG Stopped org.apache.logging.log4j.core.config.DefaultConfiguration@4310d43 OK 2024-11-24 07:35:07,240 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6 2024-11-24 07:35:07,242 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=StatusLogger 2024-11-24 07:35:07,243 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=ContextSelector 2024-11-24 07:35:07,244 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name= 2024-11-24 07:35:07,245 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.directory 2024-11-24 07:35:07,245 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSinkAdapter 2024-11-24 07:35:07,245 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.zookeeper 2024-11-24 07:35:07,246 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.logging.TestJul2Slf4j 2024-11-24 07:35:07,246 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSystemImpl 2024-11-24 07:35:07,246 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.util.MBeans 2024-11-24 07:35:07,247 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase 2024-11-24 07:35:07,247 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop 2024-11-24 07:35:07,247 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ipc.FailedServers 2024-11-24 07:35:07,248 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.regionserver.RSRpcServices 2024-11-24 07:35:07,248 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsConfig 2024-11-24 07:35:07,248 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hbase.thirdparty.io.netty.channel 2024-11-24 07:35:07,249 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ScheduledChore 2024-11-24 07:35:07,249 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Appenders,name=Console 2024-11-24 07:35:07,251 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-24 07:35:07,251 main DEBUG Reconfiguration complete for context[name=1dbd16a6] at URI jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-logging/target/hbase-logging-3.0.0-beta-2-SNAPSHOT-tests.jar!/log4j2.properties (org.apache.logging.log4j.core.LoggerContext@12f9af83) with optional ClassLoader: null 2024-11-24 07:35:07,252 main DEBUG Shutdown hook enabled. Registering a new one. 2024-11-24 07:35:07,252 main DEBUG LoggerContext[name=1dbd16a6, org.apache.logging.log4j.core.LoggerContext@12f9af83] started OK. 2024-11-24T07:35:07,468 DEBUG [main {}] hbase.HBaseTestingUtil(323): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/089ee269-8889-d2f6-f8a2-4bb8eac365fc 2024-11-24 07:35:07,470 main DEBUG AsyncLogger.ThreadNameStrategy=UNCACHED (user specified null, default is UNCACHED) 2024-11-24 07:35:07,471 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-24T07:35:07,479 INFO [main {}] hbase.HBaseClassTestRule(94): Test class org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay timeout: 13 mins 2024-11-24T07:35:07,484 INFO [main {}] hbase.HBaseClassTestRule(94): Test class org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplayValueCompression timeout: 13 mins 2024-11-24T07:35:07,505 DEBUG [Time-limited test {}] channel.MultithreadEventLoopGroup(44): -Dio.netty.eventLoopThreads: 16 2024-11-24T07:35:07,547 DEBUG [Time-limited test {}] nio.NioEventLoop(110): -Dio.netty.noKeySetOptimization: false 2024-11-24T07:35:07,547 DEBUG [Time-limited test {}] nio.NioEventLoop(111): -Dio.netty.selectorAutoRebuildThreshold: 512 2024-11-24T07:35:07,558 INFO [Time-limited test {}] hbase.HBaseTestingUtil(805): Starting up minicluster with option: StartMiniClusterOption{numMasters=1, masterClass=null, numRegionServers=3, rsPorts=, rsClass=null, numDataNodes=3, dataNodeHosts=null, numZkServers=1, createRootDir=false, createWALDir=false} 2024-11-24T07:35:07,572 INFO [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/089ee269-8889-d2f6-f8a2-4bb8eac365fc/cluster_a7d76481-495d-c9f6-361f-c7016573445a, deleteOnExit=true 2024-11-24T07:35:07,573 INFO [Time-limited test {}] hbase.HBaseTestingUtil(818): STARTING DFS 2024-11-24T07:35:07,574 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/089ee269-8889-d2f6-f8a2-4bb8eac365fc/test.cache.data in system properties and HBase conf 2024-11-24T07:35:07,574 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/089ee269-8889-d2f6-f8a2-4bb8eac365fc/hadoop.tmp.dir in system properties and HBase conf 2024-11-24T07:35:07,575 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/089ee269-8889-d2f6-f8a2-4bb8eac365fc/hadoop.log.dir in system properties and HBase conf 2024-11-24T07:35:07,575 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/089ee269-8889-d2f6-f8a2-4bb8eac365fc/mapreduce.cluster.local.dir in system properties and HBase conf 2024-11-24T07:35:07,576 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/089ee269-8889-d2f6-f8a2-4bb8eac365fc/mapreduce.cluster.temp.dir in system properties and HBase conf 2024-11-24T07:35:07,576 INFO [Time-limited test {}] hbase.HBaseTestingUtil(738): read short circuit is OFF 2024-11-24T07:35:07,645 WARN [Time-limited test {}] util.NativeCodeLoader(60): Unable to load native-hadoop library for your platform... using builtin-java classes where applicable 2024-11-24T07:35:07,726 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-24T07:35:07,730 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/089ee269-8889-d2f6-f8a2-4bb8eac365fc/yarn.node-labels.fs-store.root-dir in system properties and HBase conf 2024-11-24T07:35:07,738 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/089ee269-8889-d2f6-f8a2-4bb8eac365fc/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf 2024-11-24T07:35:07,738 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/089ee269-8889-d2f6-f8a2-4bb8eac365fc/yarn.nodemanager.log-dirs in system properties and HBase conf 2024-11-24T07:35:07,738 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/089ee269-8889-d2f6-f8a2-4bb8eac365fc/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-11-24T07:35:07,739 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/089ee269-8889-d2f6-f8a2-4bb8eac365fc/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf 2024-11-24T07:35:07,739 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/089ee269-8889-d2f6-f8a2-4bb8eac365fc/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf 2024-11-24T07:35:07,740 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/089ee269-8889-d2f6-f8a2-4bb8eac365fc/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-11-24T07:35:07,740 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/089ee269-8889-d2f6-f8a2-4bb8eac365fc/dfs.journalnode.edits.dir in system properties and HBase conf 2024-11-24T07:35:07,741 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/089ee269-8889-d2f6-f8a2-4bb8eac365fc/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf 2024-11-24T07:35:07,741 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/089ee269-8889-d2f6-f8a2-4bb8eac365fc/nfs.dump.dir in system properties and HBase conf 2024-11-24T07:35:07,741 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/089ee269-8889-d2f6-f8a2-4bb8eac365fc/java.io.tmpdir in system properties and HBase conf 2024-11-24T07:35:07,742 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/089ee269-8889-d2f6-f8a2-4bb8eac365fc/dfs.journalnode.edits.dir in system properties and HBase conf 2024-11-24T07:35:07,742 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/089ee269-8889-d2f6-f8a2-4bb8eac365fc/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf 2024-11-24T07:35:07,742 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/089ee269-8889-d2f6-f8a2-4bb8eac365fc/fs.s3a.committer.staging.tmp.path in system properties and HBase conf 2024-11-24T07:35:08,777 WARN [Time-limited test {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-namenode.properties,hadoop-metrics2.properties 2024-11-24T07:35:08,844 INFO [Time-limited test {}] log.Log(170): Logging initialized @2430ms to org.eclipse.jetty.util.log.Slf4jLog 2024-11-24T07:35:08,906 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-24T07:35:08,963 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-24T07:35:08,984 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-24T07:35:08,984 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-24T07:35:08,985 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-11-24T07:35:08,998 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-24T07:35:09,002 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@4f37ffca{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/089ee269-8889-d2f6-f8a2-4bb8eac365fc/hadoop.log.dir/,AVAILABLE} 2024-11-24T07:35:09,003 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@6dc9d5c1{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-24T07:35:09,169 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@3717288f{hdfs,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/089ee269-8889-d2f6-f8a2-4bb8eac365fc/java.io.tmpdir/jetty-localhost-42117-hadoop-hdfs-3_4_1-tests_jar-_-any-4100430810307040239/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-11-24T07:35:09,175 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@4bd70930{HTTP/1.1, (http/1.1)}{localhost:42117} 2024-11-24T07:35:09,176 INFO [Time-limited test {}] server.Server(415): Started @2762ms 2024-11-24T07:35:09,753 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-24T07:35:09,760 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-24T07:35:09,761 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-24T07:35:09,761 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-24T07:35:09,762 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-11-24T07:35:09,762 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@cf5a85e{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/089ee269-8889-d2f6-f8a2-4bb8eac365fc/hadoop.log.dir/,AVAILABLE} 2024-11-24T07:35:09,763 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@4a359997{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-24T07:35:09,862 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@330740de{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/089ee269-8889-d2f6-f8a2-4bb8eac365fc/java.io.tmpdir/jetty-localhost-44627-hadoop-hdfs-3_4_1-tests_jar-_-any-2918863162160343331/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-24T07:35:09,863 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@7b24cab9{HTTP/1.1, (http/1.1)}{localhost:44627} 2024-11-24T07:35:09,863 INFO [Time-limited test {}] server.Server(415): Started @3449ms 2024-11-24T07:35:09,909 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-11-24T07:35:10,018 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-24T07:35:10,024 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-24T07:35:10,027 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-24T07:35:10,027 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-24T07:35:10,027 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-11-24T07:35:10,028 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@46b092e1{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/089ee269-8889-d2f6-f8a2-4bb8eac365fc/hadoop.log.dir/,AVAILABLE} 2024-11-24T07:35:10,028 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@5cc2d6b{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-24T07:35:10,150 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@7bd427b8{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/089ee269-8889-d2f6-f8a2-4bb8eac365fc/java.io.tmpdir/jetty-localhost-33791-hadoop-hdfs-3_4_1-tests_jar-_-any-12003529435568982083/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-24T07:35:10,151 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@6915083f{HTTP/1.1, (http/1.1)}{localhost:33791} 2024-11-24T07:35:10,151 INFO [Time-limited test {}] server.Server(415): Started @3737ms 2024-11-24T07:35:10,153 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-11-24T07:35:10,218 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-24T07:35:10,225 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-24T07:35:10,226 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-24T07:35:10,227 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-24T07:35:10,227 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-11-24T07:35:10,228 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@2a6d5e13{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/089ee269-8889-d2f6-f8a2-4bb8eac365fc/hadoop.log.dir/,AVAILABLE} 2024-11-24T07:35:10,229 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@f9972d0{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-24T07:35:10,355 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@35f1150e{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/089ee269-8889-d2f6-f8a2-4bb8eac365fc/java.io.tmpdir/jetty-localhost-34123-hadoop-hdfs-3_4_1-tests_jar-_-any-8184459595729188222/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-24T07:35:10,356 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@13a77e13{HTTP/1.1, (http/1.1)}{localhost:34123} 2024-11-24T07:35:10,356 INFO [Time-limited test {}] server.Server(415): Started @3943ms 2024-11-24T07:35:10,358 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-11-24T07:35:11,684 WARN [Thread-125 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/089ee269-8889-d2f6-f8a2-4bb8eac365fc/cluster_a7d76481-495d-c9f6-361f-c7016573445a/data/data4/current/BP-1337235608-172.17.0.3-1732433708251/current, will proceed with Du for space computation calculation, 2024-11-24T07:35:11,684 WARN [Thread-124 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/089ee269-8889-d2f6-f8a2-4bb8eac365fc/cluster_a7d76481-495d-c9f6-361f-c7016573445a/data/data3/current/BP-1337235608-172.17.0.3-1732433708251/current, will proceed with Du for space computation calculation, 2024-11-24T07:35:11,684 WARN [Thread-123 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/089ee269-8889-d2f6-f8a2-4bb8eac365fc/cluster_a7d76481-495d-c9f6-361f-c7016573445a/data/data2/current/BP-1337235608-172.17.0.3-1732433708251/current, will proceed with Du for space computation calculation, 2024-11-24T07:35:11,684 WARN [Thread-122 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/089ee269-8889-d2f6-f8a2-4bb8eac365fc/cluster_a7d76481-495d-c9f6-361f-c7016573445a/data/data1/current/BP-1337235608-172.17.0.3-1732433708251/current, will proceed with Du for space computation calculation, 2024-11-24T07:35:11,715 WARN [Thread-58 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-11-24T07:35:11,715 WARN [Thread-81 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-11-24T07:35:11,759 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x345b59a9a005c71d with lease ID 0xadfd12e1e0efd314: Processing first storage report for DS-467ee9a1-f3ca-4279-9b61-69a2a6a0308f from datanode DatanodeRegistration(127.0.0.1:42685, datanodeUuid=36f029f0-d600-4005-8031-fb9832321be4, infoPort=35537, infoSecurePort=0, ipcPort=36885, storageInfo=lv=-57;cid=testClusterID;nsid=1417572593;c=1732433708251) 2024-11-24T07:35:11,761 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x345b59a9a005c71d with lease ID 0xadfd12e1e0efd314: from storage DS-467ee9a1-f3ca-4279-9b61-69a2a6a0308f node DatanodeRegistration(127.0.0.1:42685, datanodeUuid=36f029f0-d600-4005-8031-fb9832321be4, infoPort=35537, infoSecurePort=0, ipcPort=36885, storageInfo=lv=-57;cid=testClusterID;nsid=1417572593;c=1732433708251), blocks: 0, hasStaleStorage: true, processing time: 2 msecs, invalidatedBlocks: 0 2024-11-24T07:35:11,761 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xa65afda1771f8ef6 with lease ID 0xadfd12e1e0efd313: Processing first storage report for DS-b0260eac-1380-4e84-bf22-a60109fdf5f6 from datanode DatanodeRegistration(127.0.0.1:41161, datanodeUuid=91e56bd8-6eba-48c1-9801-3407dc9994e7, infoPort=40647, infoSecurePort=0, ipcPort=43281, storageInfo=lv=-57;cid=testClusterID;nsid=1417572593;c=1732433708251) 2024-11-24T07:35:11,761 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xa65afda1771f8ef6 with lease ID 0xadfd12e1e0efd313: from storage DS-b0260eac-1380-4e84-bf22-a60109fdf5f6 node DatanodeRegistration(127.0.0.1:41161, datanodeUuid=91e56bd8-6eba-48c1-9801-3407dc9994e7, infoPort=40647, infoSecurePort=0, ipcPort=43281, storageInfo=lv=-57;cid=testClusterID;nsid=1417572593;c=1732433708251), blocks: 0, hasStaleStorage: true, processing time: 1 msecs, invalidatedBlocks: 0 2024-11-24T07:35:11,761 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x345b59a9a005c71d with lease ID 0xadfd12e1e0efd314: Processing first storage report for DS-d3f904ba-6f32-4e1e-bbb8-7701027fe269 from datanode DatanodeRegistration(127.0.0.1:42685, datanodeUuid=36f029f0-d600-4005-8031-fb9832321be4, infoPort=35537, infoSecurePort=0, ipcPort=36885, storageInfo=lv=-57;cid=testClusterID;nsid=1417572593;c=1732433708251) 2024-11-24T07:35:11,762 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x345b59a9a005c71d with lease ID 0xadfd12e1e0efd314: from storage DS-d3f904ba-6f32-4e1e-bbb8-7701027fe269 node DatanodeRegistration(127.0.0.1:42685, datanodeUuid=36f029f0-d600-4005-8031-fb9832321be4, infoPort=35537, infoSecurePort=0, ipcPort=36885, storageInfo=lv=-57;cid=testClusterID;nsid=1417572593;c=1732433708251), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-24T07:35:11,762 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xa65afda1771f8ef6 with lease ID 0xadfd12e1e0efd313: Processing first storage report for DS-0d4d1b85-c997-4774-ad62-72fd86b37525 from datanode DatanodeRegistration(127.0.0.1:41161, datanodeUuid=91e56bd8-6eba-48c1-9801-3407dc9994e7, infoPort=40647, infoSecurePort=0, ipcPort=43281, storageInfo=lv=-57;cid=testClusterID;nsid=1417572593;c=1732433708251) 2024-11-24T07:35:11,762 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xa65afda1771f8ef6 with lease ID 0xadfd12e1e0efd313: from storage DS-0d4d1b85-c997-4774-ad62-72fd86b37525 node DatanodeRegistration(127.0.0.1:41161, datanodeUuid=91e56bd8-6eba-48c1-9801-3407dc9994e7, infoPort=40647, infoSecurePort=0, ipcPort=43281, storageInfo=lv=-57;cid=testClusterID;nsid=1417572593;c=1732433708251), blocks: 0, hasStaleStorage: false, processing time: 1 msecs, invalidatedBlocks: 0 2024-11-24T07:35:11,820 WARN [Thread-143 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/089ee269-8889-d2f6-f8a2-4bb8eac365fc/cluster_a7d76481-495d-c9f6-361f-c7016573445a/data/data6/current/BP-1337235608-172.17.0.3-1732433708251/current, will proceed with Du for space computation calculation, 2024-11-24T07:35:11,820 WARN [Thread-142 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/089ee269-8889-d2f6-f8a2-4bb8eac365fc/cluster_a7d76481-495d-c9f6-361f-c7016573445a/data/data5/current/BP-1337235608-172.17.0.3-1732433708251/current, will proceed with Du for space computation calculation, 2024-11-24T07:35:11,846 WARN [Thread-103 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-11-24T07:35:11,851 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x84225ecbbefc318e with lease ID 0xadfd12e1e0efd315: Processing first storage report for DS-a67a12dc-5f72-4d11-ac5a-3e225e33c62a from datanode DatanodeRegistration(127.0.0.1:41971, datanodeUuid=3e29c02d-96eb-4f29-acd8-ca7a6a190670, infoPort=37111, infoSecurePort=0, ipcPort=44209, storageInfo=lv=-57;cid=testClusterID;nsid=1417572593;c=1732433708251) 2024-11-24T07:35:11,852 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x84225ecbbefc318e with lease ID 0xadfd12e1e0efd315: from storage DS-a67a12dc-5f72-4d11-ac5a-3e225e33c62a node DatanodeRegistration(127.0.0.1:41971, datanodeUuid=3e29c02d-96eb-4f29-acd8-ca7a6a190670, infoPort=37111, infoSecurePort=0, ipcPort=44209, storageInfo=lv=-57;cid=testClusterID;nsid=1417572593;c=1732433708251), blocks: 0, hasStaleStorage: true, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-24T07:35:11,852 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x84225ecbbefc318e with lease ID 0xadfd12e1e0efd315: Processing first storage report for DS-a951ed8e-3c3d-41b9-bb0f-6f5c406dbda4 from datanode DatanodeRegistration(127.0.0.1:41971, datanodeUuid=3e29c02d-96eb-4f29-acd8-ca7a6a190670, infoPort=37111, infoSecurePort=0, ipcPort=44209, storageInfo=lv=-57;cid=testClusterID;nsid=1417572593;c=1732433708251) 2024-11-24T07:35:11,852 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x84225ecbbefc318e with lease ID 0xadfd12e1e0efd315: from storage DS-a951ed8e-3c3d-41b9-bb0f-6f5c406dbda4 node DatanodeRegistration(127.0.0.1:41971, datanodeUuid=3e29c02d-96eb-4f29-acd8-ca7a6a190670, infoPort=37111, infoSecurePort=0, ipcPort=44209, storageInfo=lv=-57;cid=testClusterID;nsid=1417572593;c=1732433708251), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-24T07:35:11,876 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(631): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/089ee269-8889-d2f6-f8a2-4bb8eac365fc 2024-11-24T07:35:11,951 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/089ee269-8889-d2f6-f8a2-4bb8eac365fc/cluster_a7d76481-495d-c9f6-361f-c7016573445a/zookeeper_0, clientPort=49244, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/089ee269-8889-d2f6-f8a2-4bb8eac365fc/cluster_a7d76481-495d-c9f6-361f-c7016573445a/zookeeper_0/version-2, dataDirSize=457 dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/089ee269-8889-d2f6-f8a2-4bb8eac365fc/cluster_a7d76481-495d-c9f6-361f-c7016573445a/zookeeper_0/version-2, dataLogSize=457 tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0 2024-11-24T07:35:11,961 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=49244 2024-11-24T07:35:11,970 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-24T07:35:11,972 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-24T07:35:12,160 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41971 is added to blk_1073741825_1001 (size=7) 2024-11-24T07:35:12,161 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41161 is added to blk_1073741825_1001 (size=7) 2024-11-24T07:35:12,161 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42685 is added to blk_1073741825_1001 (size=7) 2024-11-24T07:35:12,578 INFO [Time-limited test {}] util.FSUtils(489): Created version file at hdfs://localhost:33505/user/jenkins/test-data/e37915a4-8313-e328-91fb-37d541e8b214 with version=8 2024-11-24T07:35:12,578 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1139): Setting hbase.fs.tmp.dir to hdfs://localhost:33505/user/jenkins/test-data/e37915a4-8313-e328-91fb-37d541e8b214/hbase-staging 2024-11-24T07:35:12,864 INFO [Time-limited test {}] client.ConnectionUtils(128): master/1ab588ae992e:0 server-side Connection retries=45 2024-11-24T07:35:12,872 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-24T07:35:12,873 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-24T07:35:12,877 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-24T07:35:12,877 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-24T07:35:12,877 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-24T07:35:13,002 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService 2024-11-24T07:35:13,053 INFO [Time-limited test {}] metrics.MetricRegistriesLoader(60): Loaded MetricRegistries class org.apache.hadoop.hbase.metrics.impl.MetricRegistriesImpl 2024-11-24T07:35:13,061 DEBUG [Time-limited test {}] util.ClassSize(228): Using Unsafe to estimate memory layout 2024-11-24T07:35:13,064 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-24T07:35:13,086 DEBUG [Time-limited test {}] channel.DefaultChannelId(84): -Dio.netty.processId: 92639 (auto-detected) 2024-11-24T07:35:13,087 DEBUG [Time-limited test {}] channel.DefaultChannelId(106): -Dio.netty.machineId: 02:42:ac:ff:fe:11:00:03 (auto-detected) 2024-11-24T07:35:13,103 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.3:37719 2024-11-24T07:35:13,121 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:37719 connecting to ZooKeeper ensemble=127.0.0.1:49244 2024-11-24T07:35:13,268 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:377190x0, quorum=127.0.0.1:49244, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-24T07:35:13,270 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:37719-0x1016bc48eea0000 connected 2024-11-24T07:35:13,364 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-24T07:35:13,367 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-24T07:35:13,377 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:37719-0x1016bc48eea0000, quorum=127.0.0.1:49244, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-24T07:35:13,381 INFO [Time-limited test {}] master.HMaster(525): hbase.rootdir=hdfs://localhost:33505/user/jenkins/test-data/e37915a4-8313-e328-91fb-37d541e8b214, hbase.cluster.distributed=false 2024-11-24T07:35:13,402 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:37719-0x1016bc48eea0000, quorum=127.0.0.1:49244, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-24T07:35:13,406 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=37719 2024-11-24T07:35:13,406 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=37719 2024-11-24T07:35:13,407 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=37719 2024-11-24T07:35:13,407 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=37719 2024-11-24T07:35:13,408 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=37719 2024-11-24T07:35:13,505 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/1ab588ae992e:0 server-side Connection retries=45 2024-11-24T07:35:13,506 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-24T07:35:13,506 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-24T07:35:13,507 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-24T07:35:13,507 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-24T07:35:13,507 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-24T07:35:13,509 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-24T07:35:13,512 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-24T07:35:13,513 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.3:46433 2024-11-24T07:35:13,515 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:46433 connecting to ZooKeeper ensemble=127.0.0.1:49244 2024-11-24T07:35:13,516 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-24T07:35:13,520 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-24T07:35:13,538 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:464330x0, quorum=127.0.0.1:49244, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-24T07:35:13,538 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:464330x0, quorum=127.0.0.1:49244, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-24T07:35:13,539 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:46433-0x1016bc48eea0001 connected 2024-11-24T07:35:13,543 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-24T07:35:13,551 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-24T07:35:13,554 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:46433-0x1016bc48eea0001, quorum=127.0.0.1:49244, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-24T07:35:13,561 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:46433-0x1016bc48eea0001, quorum=127.0.0.1:49244, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-24T07:35:13,562 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=46433 2024-11-24T07:35:13,562 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=46433 2024-11-24T07:35:13,563 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=46433 2024-11-24T07:35:13,563 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=46433 2024-11-24T07:35:13,563 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=46433 2024-11-24T07:35:13,580 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/1ab588ae992e:0 server-side Connection retries=45 2024-11-24T07:35:13,580 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-24T07:35:13,580 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-24T07:35:13,580 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-24T07:35:13,581 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-24T07:35:13,581 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-24T07:35:13,581 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-24T07:35:13,581 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-24T07:35:13,582 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.3:40419 2024-11-24T07:35:13,584 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:40419 connecting to ZooKeeper ensemble=127.0.0.1:49244 2024-11-24T07:35:13,584 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-24T07:35:13,588 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-24T07:35:13,604 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:404190x0, quorum=127.0.0.1:49244, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-24T07:35:13,605 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:404190x0, quorum=127.0.0.1:49244, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-24T07:35:13,605 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:40419-0x1016bc48eea0002 connected 2024-11-24T07:35:13,606 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-24T07:35:13,607 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-24T07:35:13,609 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:40419-0x1016bc48eea0002, quorum=127.0.0.1:49244, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-24T07:35:13,611 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:40419-0x1016bc48eea0002, quorum=127.0.0.1:49244, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-24T07:35:13,612 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=40419 2024-11-24T07:35:13,613 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=40419 2024-11-24T07:35:13,615 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=40419 2024-11-24T07:35:13,616 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=40419 2024-11-24T07:35:13,617 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=40419 2024-11-24T07:35:13,633 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/1ab588ae992e:0 server-side Connection retries=45 2024-11-24T07:35:13,633 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-24T07:35:13,634 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-24T07:35:13,634 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-24T07:35:13,634 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-24T07:35:13,634 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-24T07:35:13,634 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-24T07:35:13,634 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-24T07:35:13,635 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.3:40611 2024-11-24T07:35:13,637 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:40611 connecting to ZooKeeper ensemble=127.0.0.1:49244 2024-11-24T07:35:13,638 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-24T07:35:13,641 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-24T07:35:13,657 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:406110x0, quorum=127.0.0.1:49244, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-24T07:35:13,658 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:406110x0, quorum=127.0.0.1:49244, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-24T07:35:13,658 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:40611-0x1016bc48eea0003 connected 2024-11-24T07:35:13,658 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-24T07:35:13,659 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-24T07:35:13,660 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:40611-0x1016bc48eea0003, quorum=127.0.0.1:49244, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-24T07:35:13,662 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:40611-0x1016bc48eea0003, quorum=127.0.0.1:49244, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-24T07:35:13,663 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=40611 2024-11-24T07:35:13,664 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=40611 2024-11-24T07:35:13,664 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=40611 2024-11-24T07:35:13,664 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=40611 2024-11-24T07:35:13,667 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=40611 2024-11-24T07:35:13,680 DEBUG [M:0;1ab588ae992e:37719 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;1ab588ae992e:37719 2024-11-24T07:35:13,681 INFO [master/1ab588ae992e:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/1ab588ae992e,37719,1732433712713 2024-11-24T07:35:13,696 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46433-0x1016bc48eea0001, quorum=127.0.0.1:49244, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-24T07:35:13,696 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40611-0x1016bc48eea0003, quorum=127.0.0.1:49244, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-24T07:35:13,696 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40419-0x1016bc48eea0002, quorum=127.0.0.1:49244, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-24T07:35:13,696 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37719-0x1016bc48eea0000, quorum=127.0.0.1:49244, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-24T07:35:13,699 DEBUG [master/1ab588ae992e:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:37719-0x1016bc48eea0000, quorum=127.0.0.1:49244, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/1ab588ae992e,37719,1732433712713 2024-11-24T07:35:13,727 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40419-0x1016bc48eea0002, quorum=127.0.0.1:49244, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-24T07:35:13,727 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37719-0x1016bc48eea0000, quorum=127.0.0.1:49244, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-24T07:35:13,727 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40611-0x1016bc48eea0003, quorum=127.0.0.1:49244, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-24T07:35:13,727 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40419-0x1016bc48eea0002, quorum=127.0.0.1:49244, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-24T07:35:13,727 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46433-0x1016bc48eea0001, quorum=127.0.0.1:49244, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-24T07:35:13,727 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40611-0x1016bc48eea0003, quorum=127.0.0.1:49244, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-24T07:35:13,727 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46433-0x1016bc48eea0001, quorum=127.0.0.1:49244, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-24T07:35:13,728 DEBUG [master/1ab588ae992e:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:37719-0x1016bc48eea0000, quorum=127.0.0.1:49244, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-11-24T07:35:13,729 INFO [master/1ab588ae992e:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/1ab588ae992e,37719,1732433712713 from backup master directory 2024-11-24T07:35:13,741 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40419-0x1016bc48eea0002, quorum=127.0.0.1:49244, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-24T07:35:13,741 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37719-0x1016bc48eea0000, quorum=127.0.0.1:49244, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/1ab588ae992e,37719,1732433712713 2024-11-24T07:35:13,741 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40611-0x1016bc48eea0003, quorum=127.0.0.1:49244, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-24T07:35:13,741 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46433-0x1016bc48eea0001, quorum=127.0.0.1:49244, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-24T07:35:13,741 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37719-0x1016bc48eea0000, quorum=127.0.0.1:49244, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-24T07:35:13,742 WARN [master/1ab588ae992e:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-24T07:35:13,742 INFO [master/1ab588ae992e:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=1ab588ae992e,37719,1732433712713 2024-11-24T07:35:13,745 INFO [master/1ab588ae992e:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating data MemStoreChunkPool with chunk size 2 MB, max count 396, initial count 0 2024-11-24T07:35:13,746 INFO [master/1ab588ae992e:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating index MemStoreChunkPool with chunk size 204.80 KB, max count 440, initial count 0 2024-11-24T07:35:13,805 DEBUG [master/1ab588ae992e:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [hdfs://localhost:33505/user/jenkins/test-data/e37915a4-8313-e328-91fb-37d541e8b214/hbase.id] with ID: c8d56efe-e808-4116-9fd2-fabfbc66fe32 2024-11-24T07:35:13,805 DEBUG [master/1ab588ae992e:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: hdfs://localhost:33505/user/jenkins/test-data/e37915a4-8313-e328-91fb-37d541e8b214/.tmp/hbase.id 2024-11-24T07:35:13,818 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41971 is added to blk_1073741826_1002 (size=42) 2024-11-24T07:35:13,819 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41161 is added to blk_1073741826_1002 (size=42) 2024-11-24T07:35:13,819 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42685 is added to blk_1073741826_1002 (size=42) 2024-11-24T07:35:13,820 DEBUG [master/1ab588ae992e:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [hdfs://localhost:33505/user/jenkins/test-data/e37915a4-8313-e328-91fb-37d541e8b214/.tmp/hbase.id]:[hdfs://localhost:33505/user/jenkins/test-data/e37915a4-8313-e328-91fb-37d541e8b214/hbase.id] 2024-11-24T07:35:13,862 INFO [master/1ab588ae992e:0:becomeActiveMaster {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-24T07:35:13,866 INFO [master/1ab588ae992e:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem. 2024-11-24T07:35:13,883 INFO [master/1ab588ae992e:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 16ms. 2024-11-24T07:35:13,895 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40611-0x1016bc48eea0003, quorum=127.0.0.1:49244, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-24T07:35:13,895 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40419-0x1016bc48eea0002, quorum=127.0.0.1:49244, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-24T07:35:13,895 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37719-0x1016bc48eea0000, quorum=127.0.0.1:49244, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-24T07:35:13,895 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46433-0x1016bc48eea0001, quorum=127.0.0.1:49244, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-24T07:35:13,916 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41971 is added to blk_1073741827_1003 (size=196) 2024-11-24T07:35:13,916 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41161 is added to blk_1073741827_1003 (size=196) 2024-11-24T07:35:13,917 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42685 is added to blk_1073741827_1003 (size=196) 2024-11-24T07:35:13,934 INFO [master/1ab588ae992e:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-24T07:35:13,936 INFO [master/1ab588ae992e:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2024-11-24T07:35:13,942 INFO [master/1ab588ae992e:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-24T07:35:13,972 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41971 is added to blk_1073741828_1004 (size=1189) 2024-11-24T07:35:13,973 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41161 is added to blk_1073741828_1004 (size=1189) 2024-11-24T07:35:13,973 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42685 is added to blk_1073741828_1004 (size=1189) 2024-11-24T07:35:13,989 INFO [master/1ab588ae992e:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir hdfs://localhost:33505/user/jenkins/test-data/e37915a4-8313-e328-91fb-37d541e8b214/MasterData/data/master/store 2024-11-24T07:35:14,012 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41971 is added to blk_1073741829_1005 (size=34) 2024-11-24T07:35:14,013 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42685 is added to blk_1073741829_1005 (size=34) 2024-11-24T07:35:14,013 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41161 is added to blk_1073741829_1005 (size=34) 2024-11-24T07:35:14,017 INFO [master/1ab588ae992e:0:becomeActiveMaster {}] throttle.StoreHotnessProtector(112): StoreHotnessProtector is disabled. Set hbase.region.store.parallel.put.limit > 0 to enable, which may help mitigate load under heavy write pressure. 2024-11-24T07:35:14,022 DEBUG [master/1ab588ae992e:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T07:35:14,023 DEBUG [master/1ab588ae992e:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-24T07:35:14,024 INFO [master/1ab588ae992e:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-24T07:35:14,024 DEBUG [master/1ab588ae992e:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-24T07:35:14,026 DEBUG [master/1ab588ae992e:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-24T07:35:14,026 DEBUG [master/1ab588ae992e:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-24T07:35:14,027 INFO [master/1ab588ae992e:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-24T07:35:14,028 DEBUG [master/1ab588ae992e:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1732433714023Disabling compacts and flushes for region at 1732433714023Disabling writes for close at 1732433714026 (+3 ms)Writing region close event to WAL at 1732433714027 (+1 ms)Closed at 1732433714027 2024-11-24T07:35:14,031 WARN [master/1ab588ae992e:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: hdfs://localhost:33505/user/jenkins/test-data/e37915a4-8313-e328-91fb-37d541e8b214/MasterData/data/master/store/.initializing 2024-11-24T07:35:14,031 DEBUG [master/1ab588ae992e:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=hdfs://localhost:33505/user/jenkins/test-data/e37915a4-8313-e328-91fb-37d541e8b214/MasterData/WALs/1ab588ae992e,37719,1732433712713 2024-11-24T07:35:14,040 INFO [master/1ab588ae992e:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-24T07:35:14,056 INFO [master/1ab588ae992e:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=1ab588ae992e%2C37719%2C1732433712713, suffix=, logDir=hdfs://localhost:33505/user/jenkins/test-data/e37915a4-8313-e328-91fb-37d541e8b214/MasterData/WALs/1ab588ae992e,37719,1732433712713, archiveDir=hdfs://localhost:33505/user/jenkins/test-data/e37915a4-8313-e328-91fb-37d541e8b214/MasterData/oldWALs, maxLogs=10 2024-11-24T07:35:14,102 DEBUG [master/1ab588ae992e:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/e37915a4-8313-e328-91fb-37d541e8b214/MasterData/WALs/1ab588ae992e,37719,1732433712713/1ab588ae992e%2C37719%2C1732433712713.1732433714062, exclude list is [], retry=0 2024-11-24T07:35:14,120 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(244): No decryptEncryptedDataEncryptionKey method in DFSClient, should be hadoop version with HDFS-12396 java.lang.NoSuchMethodException: org.apache.hadoop.hdfs.DFSClient.decryptEncryptedDataEncryptionKey(org.apache.hadoop.fs.FileEncryptionInfo) at java.lang.Class.getDeclaredMethod(Class.java:2675) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelperWithoutHDFS12396(FanOutOneBlockAsyncDFSOutputSaslHelper.java:183) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelper(FanOutOneBlockAsyncDFSOutputSaslHelper.java:242) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.(FanOutOneBlockAsyncDFSOutputSaslHelper.java:253) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.initialize(FanOutOneBlockAsyncDFSOutputHelper.java:413) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper$5.operationComplete(FanOutOneBlockAsyncDFSOutputHelper.java:472) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper$5.operationComplete(FanOutOneBlockAsyncDFSOutputHelper.java:467) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.NettyFutureUtils.lambda$addListener$0(NettyFutureUtils.java:56) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setSuccess0(DefaultPromise.java:625) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.trySuccess(DefaultPromise.java:105) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPromise.trySuccess(DefaultChannelPromise.java:84) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.fulfillConnectPromise(AbstractEpollChannel.java:658) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:696) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:491) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T07:35:14,125 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41161,DS-b0260eac-1380-4e84-bf22-a60109fdf5f6,DISK] 2024-11-24T07:35:14,125 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41971,DS-a67a12dc-5f72-4d11-ac5a-3e225e33c62a,DISK] 2024-11-24T07:35:14,125 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:42685,DS-467ee9a1-f3ca-4279-9b61-69a2a6a0308f,DISK] 2024-11-24T07:35:14,128 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.ProtobufDecoder(117): Hadoop 3.3 and above shades protobuf. 2024-11-24T07:35:14,197 INFO [master/1ab588ae992e:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/e37915a4-8313-e328-91fb-37d541e8b214/MasterData/WALs/1ab588ae992e,37719,1732433712713/1ab588ae992e%2C37719%2C1732433712713.1732433714062 2024-11-24T07:35:14,198 DEBUG [master/1ab588ae992e:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:37111:37111),(127.0.0.1/127.0.0.1:35537:35537),(127.0.0.1/127.0.0.1:40647:40647)] 2024-11-24T07:35:14,199 DEBUG [master/1ab588ae992e:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2024-11-24T07:35:14,200 DEBUG [master/1ab588ae992e:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T07:35:14,204 DEBUG [master/1ab588ae992e:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2024-11-24T07:35:14,205 DEBUG [master/1ab588ae992e:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2024-11-24T07:35:14,239 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2024-11-24T07:35:14,262 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2024-11-24T07:35:14,265 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T07:35:14,268 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-24T07:35:14,268 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2024-11-24T07:35:14,272 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2024-11-24T07:35:14,272 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T07:35:14,273 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T07:35:14,273 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2024-11-24T07:35:14,276 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2024-11-24T07:35:14,276 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T07:35:14,277 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T07:35:14,277 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2024-11-24T07:35:14,280 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2024-11-24T07:35:14,280 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T07:35:14,281 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T07:35:14,282 DEBUG [master/1ab588ae992e:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682 2024-11-24T07:35:14,285 DEBUG [master/1ab588ae992e:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33505/user/jenkins/test-data/e37915a4-8313-e328-91fb-37d541e8b214/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-24T07:35:14,286 DEBUG [master/1ab588ae992e:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33505/user/jenkins/test-data/e37915a4-8313-e328-91fb-37d541e8b214/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-24T07:35:14,291 DEBUG [master/1ab588ae992e:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682 2024-11-24T07:35:14,292 DEBUG [master/1ab588ae992e:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682 2024-11-24T07:35:14,295 DEBUG [master/1ab588ae992e:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-24T07:35:14,298 DEBUG [master/1ab588ae992e:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2024-11-24T07:35:14,302 DEBUG [master/1ab588ae992e:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:33505/user/jenkins/test-data/e37915a4-8313-e328-91fb-37d541e8b214/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-24T07:35:14,304 INFO [master/1ab588ae992e:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=68081691, jitterRate=0.014496251940727234}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-24T07:35:14,312 DEBUG [master/1ab588ae992e:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682: Writing region info on filesystem at 1732433714217Initializing all the Stores at 1732433714219 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732433714220 (+1 ms)Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732433714220Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732433714220Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732433714221 (+1 ms)Cleaning up temporary data from old regions at 1732433714292 (+71 ms)Region opened successfully at 1732433714312 (+20 ms) 2024-11-24T07:35:14,314 INFO [master/1ab588ae992e:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2024-11-24T07:35:14,345 DEBUG [master/1ab588ae992e:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5dd3f6f7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=1ab588ae992e/172.17.0.3:0 2024-11-24T07:35:14,372 INFO [master/1ab588ae992e:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating... 2024-11-24T07:35:14,382 INFO [master/1ab588ae992e:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5 2024-11-24T07:35:14,382 INFO [master/1ab588ae992e:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50 2024-11-24T07:35:14,384 INFO [master/1ab588ae992e:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-24T07:35:14,386 INFO [master/1ab588ae992e:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 1 msec 2024-11-24T07:35:14,390 INFO [master/1ab588ae992e:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 3 msec 2024-11-24T07:35:14,390 INFO [master/1ab588ae992e:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2024-11-24T07:35:14,413 INFO [master/1ab588ae992e:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2024-11-24T07:35:14,421 DEBUG [master/1ab588ae992e:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:37719-0x1016bc48eea0000, quorum=127.0.0.1:49244, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error) 2024-11-24T07:35:14,478 DEBUG [master/1ab588ae992e:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false 2024-11-24T07:35:14,481 INFO [master/1ab588ae992e:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2024-11-24T07:35:14,483 DEBUG [master/1ab588ae992e:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:37719-0x1016bc48eea0000, quorum=127.0.0.1:49244, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error) 2024-11-24T07:35:14,495 DEBUG [master/1ab588ae992e:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false 2024-11-24T07:35:14,497 INFO [master/1ab588ae992e:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2024-11-24T07:35:14,500 DEBUG [master/1ab588ae992e:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:37719-0x1016bc48eea0000, quorum=127.0.0.1:49244, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error) 2024-11-24T07:35:14,509 DEBUG [master/1ab588ae992e:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false 2024-11-24T07:35:14,511 DEBUG [master/1ab588ae992e:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:37719-0x1016bc48eea0000, quorum=127.0.0.1:49244, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error) 2024-11-24T07:35:14,519 DEBUG [master/1ab588ae992e:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false 2024-11-24T07:35:14,541 DEBUG [master/1ab588ae992e:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:37719-0x1016bc48eea0000, quorum=127.0.0.1:49244, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error) 2024-11-24T07:35:14,551 DEBUG [master/1ab588ae992e:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false 2024-11-24T07:35:14,562 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40419-0x1016bc48eea0002, quorum=127.0.0.1:49244, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-24T07:35:14,562 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37719-0x1016bc48eea0000, quorum=127.0.0.1:49244, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-24T07:35:14,562 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40419-0x1016bc48eea0002, quorum=127.0.0.1:49244, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-24T07:35:14,562 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40611-0x1016bc48eea0003, quorum=127.0.0.1:49244, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-24T07:35:14,562 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37719-0x1016bc48eea0000, quorum=127.0.0.1:49244, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-24T07:35:14,563 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40611-0x1016bc48eea0003, quorum=127.0.0.1:49244, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-24T07:35:14,563 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46433-0x1016bc48eea0001, quorum=127.0.0.1:49244, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-24T07:35:14,563 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46433-0x1016bc48eea0001, quorum=127.0.0.1:49244, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-24T07:35:14,567 INFO [master/1ab588ae992e:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=1ab588ae992e,37719,1732433712713, sessionid=0x1016bc48eea0000, setting cluster-up flag (Was=false) 2024-11-24T07:35:14,601 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37719-0x1016bc48eea0000, quorum=127.0.0.1:49244, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-24T07:35:14,601 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40611-0x1016bc48eea0003, quorum=127.0.0.1:49244, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-24T07:35:14,601 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40419-0x1016bc48eea0002, quorum=127.0.0.1:49244, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-24T07:35:14,601 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46433-0x1016bc48eea0001, quorum=127.0.0.1:49244, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-24T07:35:14,636 DEBUG [master/1ab588ae992e:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort 2024-11-24T07:35:14,640 DEBUG [master/1ab588ae992e:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=1ab588ae992e,37719,1732433712713 2024-11-24T07:35:14,667 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37719-0x1016bc48eea0000, quorum=127.0.0.1:49244, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-24T07:35:14,667 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40419-0x1016bc48eea0002, quorum=127.0.0.1:49244, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-24T07:35:14,667 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40611-0x1016bc48eea0003, quorum=127.0.0.1:49244, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-24T07:35:14,668 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46433-0x1016bc48eea0001, quorum=127.0.0.1:49244, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-24T07:35:14,699 DEBUG [master/1ab588ae992e:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort 2024-11-24T07:35:14,701 DEBUG [master/1ab588ae992e:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=1ab588ae992e,37719,1732433712713 2024-11-24T07:35:14,706 INFO [master/1ab588ae992e:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at hdfs://localhost:33505/user/jenkins/test-data/e37915a4-8313-e328-91fb-37d541e8b214/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again 2024-11-24T07:35:14,770 INFO [RS:2;1ab588ae992e:40611 {}] regionserver.HRegionServer(746): ClusterId : c8d56efe-e808-4116-9fd2-fabfbc66fe32 2024-11-24T07:35:14,770 INFO [RS:0;1ab588ae992e:46433 {}] regionserver.HRegionServer(746): ClusterId : c8d56efe-e808-4116-9fd2-fabfbc66fe32 2024-11-24T07:35:14,771 INFO [RS:1;1ab588ae992e:40419 {}] regionserver.HRegionServer(746): ClusterId : c8d56efe-e808-4116-9fd2-fabfbc66fe32 2024-11-24T07:35:14,773 DEBUG [RS:2;1ab588ae992e:40611 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-24T07:35:14,773 DEBUG [RS:0;1ab588ae992e:46433 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-24T07:35:14,773 DEBUG [RS:1;1ab588ae992e:40419 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-24T07:35:14,795 DEBUG [RS:0;1ab588ae992e:46433 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-24T07:35:14,795 DEBUG [RS:1;1ab588ae992e:40419 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-24T07:35:14,795 DEBUG [RS:2;1ab588ae992e:40611 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-24T07:35:14,795 DEBUG [RS:0;1ab588ae992e:46433 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-24T07:35:14,795 DEBUG [RS:2;1ab588ae992e:40611 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-24T07:35:14,795 DEBUG [RS:1;1ab588ae992e:40419 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-24T07:35:14,801 INFO [AsyncFSWAL-0-hdfs://localhost:33505/user/jenkins/test-data/e37915a4-8313-e328-91fb-37d541e8b214/MasterData-prefix:1ab588ae992e,37719,1732433712713 {}] compress.Compression(560): Loaded codec org.apache.hadoop.hbase.io.compress.ReusableStreamGzipCodec for compression algorithm GZ 2024-11-24T07:35:14,814 DEBUG [master/1ab588ae992e:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-24T07:35:14,823 INFO [master/1ab588ae992e:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2 2024-11-24T07:35:14,829 INFO [master/1ab588ae992e:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc. 2024-11-24T07:35:14,834 DEBUG [RS:0;1ab588ae992e:46433 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-24T07:35:14,834 DEBUG [RS:1;1ab588ae992e:40419 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-24T07:35:14,834 DEBUG [RS:2;1ab588ae992e:40611 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-24T07:35:14,835 DEBUG [RS:1;1ab588ae992e:40419 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6460f6b2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=1ab588ae992e/172.17.0.3:0 2024-11-24T07:35:14,835 DEBUG [RS:0;1ab588ae992e:46433 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5cc197a3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=1ab588ae992e/172.17.0.3:0 2024-11-24T07:35:14,835 DEBUG [RS:2;1ab588ae992e:40611 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@49f1360b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=1ab588ae992e/172.17.0.3:0 2024-11-24T07:35:14,836 DEBUG [master/1ab588ae992e:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: 1ab588ae992e,37719,1732433712713 Number of backup masters: 0 Number of live region servers: 0 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2024-11-24T07:35:14,843 DEBUG [master/1ab588ae992e:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/1ab588ae992e:0, corePoolSize=5, maxPoolSize=5 2024-11-24T07:35:14,843 DEBUG [master/1ab588ae992e:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/1ab588ae992e:0, corePoolSize=5, maxPoolSize=5 2024-11-24T07:35:14,843 DEBUG [master/1ab588ae992e:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/1ab588ae992e:0, corePoolSize=5, maxPoolSize=5 2024-11-24T07:35:14,843 DEBUG [master/1ab588ae992e:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/1ab588ae992e:0, corePoolSize=5, maxPoolSize=5 2024-11-24T07:35:14,843 DEBUG [master/1ab588ae992e:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/1ab588ae992e:0, corePoolSize=10, maxPoolSize=10 2024-11-24T07:35:14,844 DEBUG [master/1ab588ae992e:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/1ab588ae992e:0, corePoolSize=1, maxPoolSize=1 2024-11-24T07:35:14,844 DEBUG [master/1ab588ae992e:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/1ab588ae992e:0, corePoolSize=2, maxPoolSize=2 2024-11-24T07:35:14,844 DEBUG [master/1ab588ae992e:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/1ab588ae992e:0, corePoolSize=1, maxPoolSize=1 2024-11-24T07:35:14,855 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-24T07:35:14,855 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region 2024-11-24T07:35:14,856 DEBUG [RS:1;1ab588ae992e:40419 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:1;1ab588ae992e:40419 2024-11-24T07:35:14,857 DEBUG [RS:2;1ab588ae992e:40611 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:2;1ab588ae992e:40611 2024-11-24T07:35:14,858 DEBUG [RS:0;1ab588ae992e:46433 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;1ab588ae992e:46433 2024-11-24T07:35:14,859 INFO [master/1ab588ae992e:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732433744859 2024-11-24T07:35:14,860 INFO [RS:1;1ab588ae992e:40419 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-24T07:35:14,860 INFO [RS:0;1ab588ae992e:46433 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-24T07:35:14,860 INFO [RS:2;1ab588ae992e:40611 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-24T07:35:14,860 INFO [RS:1;1ab588ae992e:40419 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-24T07:35:14,860 INFO [RS:0;1ab588ae992e:46433 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-24T07:35:14,860 INFO [RS:2;1ab588ae992e:40611 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-24T07:35:14,860 DEBUG [RS:2;1ab588ae992e:40611 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-24T07:35:14,860 DEBUG [RS:1;1ab588ae992e:40419 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-24T07:35:14,860 DEBUG [RS:0;1ab588ae992e:46433 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-24T07:35:14,861 INFO [master/1ab588ae992e:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2024-11-24T07:35:14,862 INFO [master/1ab588ae992e:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2024-11-24T07:35:14,863 INFO [RS:2;1ab588ae992e:40611 {}] regionserver.HRegionServer(2659): reportForDuty to master=1ab588ae992e,37719,1732433712713 with port=40611, startcode=1732433713633 2024-11-24T07:35:14,863 INFO [RS:1;1ab588ae992e:40419 {}] regionserver.HRegionServer(2659): reportForDuty to master=1ab588ae992e,37719,1732433712713 with port=40419, startcode=1732433713579 2024-11-24T07:35:14,863 INFO [RS:0;1ab588ae992e:46433 {}] regionserver.HRegionServer(2659): reportForDuty to master=1ab588ae992e,37719,1732433712713 with port=46433, startcode=1732433713473 2024-11-24T07:35:14,863 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T07:35:14,864 INFO [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2024-11-24T07:35:14,867 INFO [master/1ab588ae992e:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2024-11-24T07:35:14,867 INFO [master/1ab588ae992e:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2024-11-24T07:35:14,868 INFO [master/1ab588ae992e:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2024-11-24T07:35:14,868 INFO [master/1ab588ae992e:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2024-11-24T07:35:14,876 DEBUG [RS:0;1ab588ae992e:46433 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-24T07:35:14,876 DEBUG [RS:2;1ab588ae992e:40611 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-24T07:35:14,876 DEBUG [RS:1;1ab588ae992e:40419 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-24T07:35:14,876 INFO [master/1ab588ae992e:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-24T07:35:14,881 INFO [master/1ab588ae992e:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2024-11-24T07:35:14,882 INFO [master/1ab588ae992e:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2024-11-24T07:35:14,882 INFO [master/1ab588ae992e:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2024-11-24T07:35:14,893 INFO [master/1ab588ae992e:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2024-11-24T07:35:14,920 INFO [master/1ab588ae992e:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2024-11-24T07:35:14,923 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41161 is added to blk_1073741831_1007 (size=1321) 2024-11-24T07:35:14,923 DEBUG [master/1ab588ae992e:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/1ab588ae992e:0:becomeActiveMaster-HFileCleaner.large.0-1732433714922,5,FailOnTimeoutGroup] 2024-11-24T07:35:14,923 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41971 is added to blk_1073741831_1007 (size=1321) 2024-11-24T07:35:14,923 DEBUG [master/1ab588ae992e:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/1ab588ae992e:0:becomeActiveMaster-HFileCleaner.small.0-1732433714923,5,FailOnTimeoutGroup] 2024-11-24T07:35:14,923 INFO [master/1ab588ae992e:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-24T07:35:14,923 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42685 is added to blk_1073741831_1007 (size=1321) 2024-11-24T07:35:14,924 INFO [master/1ab588ae992e:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2024-11-24T07:35:14,925 INFO [master/1ab588ae992e:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2024-11-24T07:35:14,925 INFO [master/1ab588ae992e:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2024-11-24T07:35:14,926 INFO [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to hdfs://localhost:33505/user/jenkins/test-data/e37915a4-8313-e328-91fb-37d541e8b214/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321 2024-11-24T07:35:14,927 INFO [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:33505/user/jenkins/test-data/e37915a4-8313-e328-91fb-37d541e8b214 2024-11-24T07:35:14,941 INFO [HMaster-EventLoopGroup-2-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:39991, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins.hfs.2 (auth:SIMPLE), service=RegionServerStatusService 2024-11-24T07:35:14,941 INFO [HMaster-EventLoopGroup-2-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:44475, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=RegionServerStatusService 2024-11-24T07:35:14,941 INFO [HMaster-EventLoopGroup-2-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:42541, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins.hfs.1 (auth:SIMPLE), service=RegionServerStatusService 2024-11-24T07:35:14,947 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41161 is added to blk_1073741832_1008 (size=32) 2024-11-24T07:35:14,948 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41971 is added to blk_1073741832_1008 (size=32) 2024-11-24T07:35:14,948 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42685 is added to blk_1073741832_1008 (size=32) 2024-11-24T07:35:14,949 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=37719 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 1ab588ae992e,40419,1732433713579 2024-11-24T07:35:14,950 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T07:35:14,951 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=37719 {}] master.ServerManager(517): Registering regionserver=1ab588ae992e,40419,1732433713579 2024-11-24T07:35:14,953 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-24T07:35:14,957 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-24T07:35:14,957 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T07:35:14,958 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-24T07:35:14,959 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-24T07:35:14,962 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-24T07:35:14,962 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T07:35:14,963 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=37719 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 1ab588ae992e,46433,1732433713473 2024-11-24T07:35:14,964 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-24T07:35:14,964 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=37719 {}] master.ServerManager(517): Registering regionserver=1ab588ae992e,46433,1732433713473 2024-11-24T07:35:14,964 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-24T07:35:14,968 DEBUG [RS:1;1ab588ae992e:40419 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:33505/user/jenkins/test-data/e37915a4-8313-e328-91fb-37d541e8b214 2024-11-24T07:35:14,968 DEBUG [RS:1;1ab588ae992e:40419 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:33505 2024-11-24T07:35:14,968 DEBUG [RS:1;1ab588ae992e:40419 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-24T07:35:14,968 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-24T07:35:14,968 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T07:35:14,970 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-24T07:35:14,970 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-24T07:35:14,970 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=37719 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 1ab588ae992e,40611,1732433713633 2024-11-24T07:35:14,970 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=37719 {}] master.ServerManager(517): Registering regionserver=1ab588ae992e,40611,1732433713633 2024-11-24T07:35:14,970 DEBUG [RS:0;1ab588ae992e:46433 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:33505/user/jenkins/test-data/e37915a4-8313-e328-91fb-37d541e8b214 2024-11-24T07:35:14,971 DEBUG [RS:0;1ab588ae992e:46433 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:33505 2024-11-24T07:35:14,971 DEBUG [RS:0;1ab588ae992e:46433 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-24T07:35:14,973 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-24T07:35:14,974 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T07:35:14,974 DEBUG [RS:2;1ab588ae992e:40611 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:33505/user/jenkins/test-data/e37915a4-8313-e328-91fb-37d541e8b214 2024-11-24T07:35:14,974 DEBUG [RS:2;1ab588ae992e:40611 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:33505 2024-11-24T07:35:14,974 DEBUG [RS:2;1ab588ae992e:40611 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-24T07:35:14,975 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-24T07:35:14,976 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-24T07:35:14,977 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33505/user/jenkins/test-data/e37915a4-8313-e328-91fb-37d541e8b214/data/hbase/meta/1588230740 2024-11-24T07:35:14,978 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33505/user/jenkins/test-data/e37915a4-8313-e328-91fb-37d541e8b214/data/hbase/meta/1588230740 2024-11-24T07:35:14,981 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-24T07:35:14,981 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-24T07:35:14,982 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-24T07:35:14,983 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37719-0x1016bc48eea0000, quorum=127.0.0.1:49244, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-24T07:35:14,985 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-24T07:35:14,990 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:33505/user/jenkins/test-data/e37915a4-8313-e328-91fb-37d541e8b214/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-24T07:35:14,991 INFO [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=67360224, jitterRate=0.003745555877685547}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-24T07:35:14,994 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740: Writing region info on filesystem at 1732433714950Initializing all the Stores at 1732433714952 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732433714953 (+1 ms)Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732433714953Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732433714953Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732433714953Cleaning up temporary data from old regions at 1732433714981 (+28 ms)Region opened successfully at 1732433714994 (+13 ms) 2024-11-24T07:35:14,994 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-24T07:35:14,995 INFO [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-24T07:35:14,995 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-24T07:35:14,995 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-24T07:35:14,995 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-24T07:35:14,996 INFO [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-24T07:35:14,996 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1732433714994Disabling compacts and flushes for region at 1732433714994Disabling writes for close at 1732433714995 (+1 ms)Writing region close event to WAL at 1732433714996 (+1 ms)Closed at 1732433714996 2024-11-24T07:35:14,999 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-24T07:35:14,999 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta 2024-11-24T07:35:15,004 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-11-24T07:35:15,011 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-24T07:35:15,014 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2024-11-24T07:35:15,028 DEBUG [RS:1;1ab588ae992e:40419 {}] zookeeper.ZKUtil(111): regionserver:40419-0x1016bc48eea0002, quorum=127.0.0.1:49244, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/1ab588ae992e,40419,1732433713579 2024-11-24T07:35:15,028 WARN [RS:1;1ab588ae992e:40419 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-24T07:35:15,029 INFO [RS:1;1ab588ae992e:40419 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-24T07:35:15,029 DEBUG [RS:1;1ab588ae992e:40419 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:33505/user/jenkins/test-data/e37915a4-8313-e328-91fb-37d541e8b214/WALs/1ab588ae992e,40419,1732433713579 2024-11-24T07:35:15,030 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [1ab588ae992e,46433,1732433713473] 2024-11-24T07:35:15,030 DEBUG [RS:0;1ab588ae992e:46433 {}] zookeeper.ZKUtil(111): regionserver:46433-0x1016bc48eea0001, quorum=127.0.0.1:49244, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/1ab588ae992e,46433,1732433713473 2024-11-24T07:35:15,030 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [1ab588ae992e,40611,1732433713633] 2024-11-24T07:35:15,030 DEBUG [RS:2;1ab588ae992e:40611 {}] zookeeper.ZKUtil(111): regionserver:40611-0x1016bc48eea0003, quorum=127.0.0.1:49244, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/1ab588ae992e,40611,1732433713633 2024-11-24T07:35:15,030 WARN [RS:0;1ab588ae992e:46433 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-24T07:35:15,030 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [1ab588ae992e,40419,1732433713579] 2024-11-24T07:35:15,031 WARN [RS:2;1ab588ae992e:40611 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-24T07:35:15,031 INFO [RS:0;1ab588ae992e:46433 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-24T07:35:15,031 INFO [RS:2;1ab588ae992e:40611 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-24T07:35:15,031 DEBUG [RS:0;1ab588ae992e:46433 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:33505/user/jenkins/test-data/e37915a4-8313-e328-91fb-37d541e8b214/WALs/1ab588ae992e,46433,1732433713473 2024-11-24T07:35:15,031 DEBUG [RS:2;1ab588ae992e:40611 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:33505/user/jenkins/test-data/e37915a4-8313-e328-91fb-37d541e8b214/WALs/1ab588ae992e,40611,1732433713633 2024-11-24T07:35:15,055 INFO [RS:0;1ab588ae992e:46433 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-24T07:35:15,055 INFO [RS:2;1ab588ae992e:40611 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-24T07:35:15,055 INFO [RS:1;1ab588ae992e:40419 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-24T07:35:15,070 INFO [RS:0;1ab588ae992e:46433 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-24T07:35:15,070 INFO [RS:2;1ab588ae992e:40611 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-24T07:35:15,071 INFO [RS:1;1ab588ae992e:40419 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-24T07:35:15,075 INFO [RS:0;1ab588ae992e:46433 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-24T07:35:15,075 INFO [RS:2;1ab588ae992e:40611 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-24T07:35:15,075 INFO [RS:1;1ab588ae992e:40419 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-24T07:35:15,075 INFO [RS:0;1ab588ae992e:46433 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-24T07:35:15,075 INFO [RS:1;1ab588ae992e:40419 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-24T07:35:15,075 INFO [RS:2;1ab588ae992e:40611 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-24T07:35:15,077 INFO [RS:0;1ab588ae992e:46433 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-24T07:35:15,080 INFO [RS:1;1ab588ae992e:40419 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-24T07:35:15,080 INFO [RS:2;1ab588ae992e:40611 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-24T07:35:15,083 INFO [RS:1;1ab588ae992e:40419 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-24T07:35:15,083 INFO [RS:2;1ab588ae992e:40611 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-24T07:35:15,083 INFO [RS:0;1ab588ae992e:46433 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-24T07:35:15,085 INFO [RS:0;1ab588ae992e:46433 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-24T07:35:15,085 INFO [RS:1;1ab588ae992e:40419 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-24T07:35:15,085 INFO [RS:2;1ab588ae992e:40611 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-24T07:35:15,085 DEBUG [RS:2;1ab588ae992e:40611 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/1ab588ae992e:0, corePoolSize=1, maxPoolSize=1 2024-11-24T07:35:15,085 DEBUG [RS:0;1ab588ae992e:46433 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/1ab588ae992e:0, corePoolSize=1, maxPoolSize=1 2024-11-24T07:35:15,085 DEBUG [RS:1;1ab588ae992e:40419 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/1ab588ae992e:0, corePoolSize=1, maxPoolSize=1 2024-11-24T07:35:15,085 DEBUG [RS:2;1ab588ae992e:40611 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/1ab588ae992e:0, corePoolSize=1, maxPoolSize=1 2024-11-24T07:35:15,085 DEBUG [RS:1;1ab588ae992e:40419 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/1ab588ae992e:0, corePoolSize=1, maxPoolSize=1 2024-11-24T07:35:15,085 DEBUG [RS:0;1ab588ae992e:46433 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/1ab588ae992e:0, corePoolSize=1, maxPoolSize=1 2024-11-24T07:35:15,085 DEBUG [RS:2;1ab588ae992e:40611 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/1ab588ae992e:0, corePoolSize=1, maxPoolSize=1 2024-11-24T07:35:15,086 DEBUG [RS:1;1ab588ae992e:40419 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/1ab588ae992e:0, corePoolSize=1, maxPoolSize=1 2024-11-24T07:35:15,086 DEBUG [RS:0;1ab588ae992e:46433 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/1ab588ae992e:0, corePoolSize=1, maxPoolSize=1 2024-11-24T07:35:15,086 DEBUG [RS:2;1ab588ae992e:40611 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/1ab588ae992e:0, corePoolSize=1, maxPoolSize=1 2024-11-24T07:35:15,086 DEBUG [RS:1;1ab588ae992e:40419 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/1ab588ae992e:0, corePoolSize=1, maxPoolSize=1 2024-11-24T07:35:15,086 DEBUG [RS:0;1ab588ae992e:46433 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/1ab588ae992e:0, corePoolSize=1, maxPoolSize=1 2024-11-24T07:35:15,086 DEBUG [RS:2;1ab588ae992e:40611 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/1ab588ae992e:0, corePoolSize=1, maxPoolSize=1 2024-11-24T07:35:15,086 DEBUG [RS:1;1ab588ae992e:40419 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/1ab588ae992e:0, corePoolSize=1, maxPoolSize=1 2024-11-24T07:35:15,086 DEBUG [RS:0;1ab588ae992e:46433 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/1ab588ae992e:0, corePoolSize=1, maxPoolSize=1 2024-11-24T07:35:15,086 DEBUG [RS:2;1ab588ae992e:40611 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/1ab588ae992e:0, corePoolSize=2, maxPoolSize=2 2024-11-24T07:35:15,086 DEBUG [RS:1;1ab588ae992e:40419 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/1ab588ae992e:0, corePoolSize=2, maxPoolSize=2 2024-11-24T07:35:15,086 DEBUG [RS:0;1ab588ae992e:46433 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/1ab588ae992e:0, corePoolSize=2, maxPoolSize=2 2024-11-24T07:35:15,086 DEBUG [RS:2;1ab588ae992e:40611 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/1ab588ae992e:0, corePoolSize=1, maxPoolSize=1 2024-11-24T07:35:15,086 DEBUG [RS:1;1ab588ae992e:40419 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/1ab588ae992e:0, corePoolSize=1, maxPoolSize=1 2024-11-24T07:35:15,086 DEBUG [RS:0;1ab588ae992e:46433 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/1ab588ae992e:0, corePoolSize=1, maxPoolSize=1 2024-11-24T07:35:15,086 DEBUG [RS:2;1ab588ae992e:40611 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/1ab588ae992e:0, corePoolSize=1, maxPoolSize=1 2024-11-24T07:35:15,086 DEBUG [RS:1;1ab588ae992e:40419 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/1ab588ae992e:0, corePoolSize=1, maxPoolSize=1 2024-11-24T07:35:15,086 DEBUG [RS:0;1ab588ae992e:46433 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/1ab588ae992e:0, corePoolSize=1, maxPoolSize=1 2024-11-24T07:35:15,086 DEBUG [RS:2;1ab588ae992e:40611 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/1ab588ae992e:0, corePoolSize=1, maxPoolSize=1 2024-11-24T07:35:15,086 DEBUG [RS:1;1ab588ae992e:40419 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/1ab588ae992e:0, corePoolSize=1, maxPoolSize=1 2024-11-24T07:35:15,086 DEBUG [RS:0;1ab588ae992e:46433 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/1ab588ae992e:0, corePoolSize=1, maxPoolSize=1 2024-11-24T07:35:15,086 DEBUG [RS:1;1ab588ae992e:40419 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/1ab588ae992e:0, corePoolSize=1, maxPoolSize=1 2024-11-24T07:35:15,086 DEBUG [RS:2;1ab588ae992e:40611 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/1ab588ae992e:0, corePoolSize=1, maxPoolSize=1 2024-11-24T07:35:15,087 DEBUG [RS:0;1ab588ae992e:46433 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/1ab588ae992e:0, corePoolSize=1, maxPoolSize=1 2024-11-24T07:35:15,087 DEBUG [RS:2;1ab588ae992e:40611 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/1ab588ae992e:0, corePoolSize=1, maxPoolSize=1 2024-11-24T07:35:15,087 DEBUG [RS:1;1ab588ae992e:40419 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/1ab588ae992e:0, corePoolSize=1, maxPoolSize=1 2024-11-24T07:35:15,087 DEBUG [RS:0;1ab588ae992e:46433 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/1ab588ae992e:0, corePoolSize=1, maxPoolSize=1 2024-11-24T07:35:15,087 DEBUG [RS:1;1ab588ae992e:40419 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/1ab588ae992e:0, corePoolSize=1, maxPoolSize=1 2024-11-24T07:35:15,087 DEBUG [RS:2;1ab588ae992e:40611 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/1ab588ae992e:0, corePoolSize=1, maxPoolSize=1 2024-11-24T07:35:15,087 DEBUG [RS:0;1ab588ae992e:46433 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/1ab588ae992e:0, corePoolSize=1, maxPoolSize=1 2024-11-24T07:35:15,087 DEBUG [RS:1;1ab588ae992e:40419 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/1ab588ae992e:0, corePoolSize=3, maxPoolSize=3 2024-11-24T07:35:15,087 DEBUG [RS:2;1ab588ae992e:40611 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/1ab588ae992e:0, corePoolSize=3, maxPoolSize=3 2024-11-24T07:35:15,087 DEBUG [RS:0;1ab588ae992e:46433 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/1ab588ae992e:0, corePoolSize=3, maxPoolSize=3 2024-11-24T07:35:15,087 DEBUG [RS:2;1ab588ae992e:40611 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/1ab588ae992e:0, corePoolSize=3, maxPoolSize=3 2024-11-24T07:35:15,087 DEBUG [RS:1;1ab588ae992e:40419 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/1ab588ae992e:0, corePoolSize=3, maxPoolSize=3 2024-11-24T07:35:15,087 DEBUG [RS:0;1ab588ae992e:46433 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/1ab588ae992e:0, corePoolSize=3, maxPoolSize=3 2024-11-24T07:35:15,092 INFO [RS:2;1ab588ae992e:40611 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-24T07:35:15,092 INFO [RS:2;1ab588ae992e:40611 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-24T07:35:15,092 INFO [RS:2;1ab588ae992e:40611 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-24T07:35:15,093 INFO [RS:2;1ab588ae992e:40611 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-24T07:35:15,093 INFO [RS:2;1ab588ae992e:40611 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-24T07:35:15,093 INFO [RS:2;1ab588ae992e:40611 {}] hbase.ChoreService(168): Chore ScheduledChore name=1ab588ae992e,40611,1732433713633-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-24T07:35:15,093 INFO [RS:1;1ab588ae992e:40419 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-24T07:35:15,093 INFO [RS:1;1ab588ae992e:40419 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-24T07:35:15,093 INFO [RS:1;1ab588ae992e:40419 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-24T07:35:15,093 INFO [RS:1;1ab588ae992e:40419 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-24T07:35:15,093 INFO [RS:0;1ab588ae992e:46433 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-24T07:35:15,093 INFO [RS:1;1ab588ae992e:40419 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-24T07:35:15,094 INFO [RS:0;1ab588ae992e:46433 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-24T07:35:15,094 INFO [RS:1;1ab588ae992e:40419 {}] hbase.ChoreService(168): Chore ScheduledChore name=1ab588ae992e,40419,1732433713579-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-24T07:35:15,094 INFO [RS:0;1ab588ae992e:46433 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-24T07:35:15,094 INFO [RS:0;1ab588ae992e:46433 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-24T07:35:15,094 INFO [RS:0;1ab588ae992e:46433 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-24T07:35:15,094 INFO [RS:0;1ab588ae992e:46433 {}] hbase.ChoreService(168): Chore ScheduledChore name=1ab588ae992e,46433,1732433713473-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-24T07:35:15,115 INFO [RS:0;1ab588ae992e:46433 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-24T07:35:15,115 INFO [RS:2;1ab588ae992e:40611 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-24T07:35:15,115 INFO [RS:1;1ab588ae992e:40419 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-24T07:35:15,117 INFO [RS:0;1ab588ae992e:46433 {}] hbase.ChoreService(168): Chore ScheduledChore name=1ab588ae992e,46433,1732433713473-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-24T07:35:15,117 INFO [RS:1;1ab588ae992e:40419 {}] hbase.ChoreService(168): Chore ScheduledChore name=1ab588ae992e,40419,1732433713579-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-24T07:35:15,117 INFO [RS:2;1ab588ae992e:40611 {}] hbase.ChoreService(168): Chore ScheduledChore name=1ab588ae992e,40611,1732433713633-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-24T07:35:15,117 INFO [RS:0;1ab588ae992e:46433 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-24T07:35:15,117 INFO [RS:1;1ab588ae992e:40419 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-24T07:35:15,117 INFO [RS:2;1ab588ae992e:40611 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-24T07:35:15,117 INFO [RS:0;1ab588ae992e:46433 {}] regionserver.Replication(171): 1ab588ae992e,46433,1732433713473 started 2024-11-24T07:35:15,117 INFO [RS:2;1ab588ae992e:40611 {}] regionserver.Replication(171): 1ab588ae992e,40611,1732433713633 started 2024-11-24T07:35:15,117 INFO [RS:1;1ab588ae992e:40419 {}] regionserver.Replication(171): 1ab588ae992e,40419,1732433713579 started 2024-11-24T07:35:15,135 INFO [RS:0;1ab588ae992e:46433 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-24T07:35:15,135 INFO [RS:1;1ab588ae992e:40419 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-24T07:35:15,136 INFO [RS:0;1ab588ae992e:46433 {}] regionserver.HRegionServer(1482): Serving as 1ab588ae992e,46433,1732433713473, RpcServer on 1ab588ae992e/172.17.0.3:46433, sessionid=0x1016bc48eea0001 2024-11-24T07:35:15,136 INFO [RS:1;1ab588ae992e:40419 {}] regionserver.HRegionServer(1482): Serving as 1ab588ae992e,40419,1732433713579, RpcServer on 1ab588ae992e/172.17.0.3:40419, sessionid=0x1016bc48eea0002 2024-11-24T07:35:15,137 DEBUG [RS:0;1ab588ae992e:46433 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-24T07:35:15,137 DEBUG [RS:1;1ab588ae992e:40419 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-24T07:35:15,137 DEBUG [RS:0;1ab588ae992e:46433 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 1ab588ae992e,46433,1732433713473 2024-11-24T07:35:15,137 DEBUG [RS:1;1ab588ae992e:40419 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 1ab588ae992e,40419,1732433713579 2024-11-24T07:35:15,137 DEBUG [RS:0;1ab588ae992e:46433 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '1ab588ae992e,46433,1732433713473' 2024-11-24T07:35:15,137 DEBUG [RS:1;1ab588ae992e:40419 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '1ab588ae992e,40419,1732433713579' 2024-11-24T07:35:15,137 DEBUG [RS:0;1ab588ae992e:46433 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-24T07:35:15,137 DEBUG [RS:1;1ab588ae992e:40419 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-24T07:35:15,138 DEBUG [RS:1;1ab588ae992e:40419 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-24T07:35:15,138 DEBUG [RS:0;1ab588ae992e:46433 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-24T07:35:15,139 DEBUG [RS:0;1ab588ae992e:46433 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-24T07:35:15,139 DEBUG [RS:1;1ab588ae992e:40419 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-24T07:35:15,139 DEBUG [RS:0;1ab588ae992e:46433 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-24T07:35:15,139 DEBUG [RS:1;1ab588ae992e:40419 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-24T07:35:15,139 DEBUG [RS:1;1ab588ae992e:40419 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 1ab588ae992e,40419,1732433713579 2024-11-24T07:35:15,139 DEBUG [RS:0;1ab588ae992e:46433 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 1ab588ae992e,46433,1732433713473 2024-11-24T07:35:15,139 DEBUG [RS:1;1ab588ae992e:40419 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '1ab588ae992e,40419,1732433713579' 2024-11-24T07:35:15,139 DEBUG [RS:0;1ab588ae992e:46433 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '1ab588ae992e,46433,1732433713473' 2024-11-24T07:35:15,139 DEBUG [RS:0;1ab588ae992e:46433 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-24T07:35:15,139 DEBUG [RS:1;1ab588ae992e:40419 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-24T07:35:15,139 DEBUG [RS:0;1ab588ae992e:46433 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-24T07:35:15,139 DEBUG [RS:1;1ab588ae992e:40419 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-24T07:35:15,140 DEBUG [RS:0;1ab588ae992e:46433 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-24T07:35:15,140 DEBUG [RS:1;1ab588ae992e:40419 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-24T07:35:15,140 INFO [RS:1;1ab588ae992e:40419 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-24T07:35:15,140 INFO [RS:0;1ab588ae992e:46433 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-24T07:35:15,140 INFO [RS:0;1ab588ae992e:46433 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-24T07:35:15,140 INFO [RS:1;1ab588ae992e:40419 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-24T07:35:15,140 INFO [RS:2;1ab588ae992e:40611 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-24T07:35:15,140 INFO [RS:2;1ab588ae992e:40611 {}] regionserver.HRegionServer(1482): Serving as 1ab588ae992e,40611,1732433713633, RpcServer on 1ab588ae992e/172.17.0.3:40611, sessionid=0x1016bc48eea0003 2024-11-24T07:35:15,141 DEBUG [RS:2;1ab588ae992e:40611 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-24T07:35:15,141 DEBUG [RS:2;1ab588ae992e:40611 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 1ab588ae992e,40611,1732433713633 2024-11-24T07:35:15,141 DEBUG [RS:2;1ab588ae992e:40611 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '1ab588ae992e,40611,1732433713633' 2024-11-24T07:35:15,141 DEBUG [RS:2;1ab588ae992e:40611 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-24T07:35:15,141 DEBUG [RS:2;1ab588ae992e:40611 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-24T07:35:15,142 DEBUG [RS:2;1ab588ae992e:40611 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-24T07:35:15,142 DEBUG [RS:2;1ab588ae992e:40611 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-24T07:35:15,142 DEBUG [RS:2;1ab588ae992e:40611 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 1ab588ae992e,40611,1732433713633 2024-11-24T07:35:15,142 DEBUG [RS:2;1ab588ae992e:40611 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '1ab588ae992e,40611,1732433713633' 2024-11-24T07:35:15,142 DEBUG [RS:2;1ab588ae992e:40611 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-24T07:35:15,143 DEBUG [RS:2;1ab588ae992e:40611 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-24T07:35:15,143 DEBUG [RS:2;1ab588ae992e:40611 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-24T07:35:15,143 INFO [RS:2;1ab588ae992e:40611 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-24T07:35:15,143 INFO [RS:2;1ab588ae992e:40611 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-24T07:35:15,165 WARN [1ab588ae992e:37719 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-24T07:35:15,247 INFO [RS:2;1ab588ae992e:40611 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-24T07:35:15,247 INFO [RS:0;1ab588ae992e:46433 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-24T07:35:15,247 INFO [RS:1;1ab588ae992e:40419 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-24T07:35:15,250 INFO [RS:2;1ab588ae992e:40611 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=1ab588ae992e%2C40611%2C1732433713633, suffix=, logDir=hdfs://localhost:33505/user/jenkins/test-data/e37915a4-8313-e328-91fb-37d541e8b214/WALs/1ab588ae992e,40611,1732433713633, archiveDir=hdfs://localhost:33505/user/jenkins/test-data/e37915a4-8313-e328-91fb-37d541e8b214/oldWALs, maxLogs=32 2024-11-24T07:35:15,250 INFO [RS:0;1ab588ae992e:46433 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=1ab588ae992e%2C46433%2C1732433713473, suffix=, logDir=hdfs://localhost:33505/user/jenkins/test-data/e37915a4-8313-e328-91fb-37d541e8b214/WALs/1ab588ae992e,46433,1732433713473, archiveDir=hdfs://localhost:33505/user/jenkins/test-data/e37915a4-8313-e328-91fb-37d541e8b214/oldWALs, maxLogs=32 2024-11-24T07:35:15,250 INFO [RS:1;1ab588ae992e:40419 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=1ab588ae992e%2C40419%2C1732433713579, suffix=, logDir=hdfs://localhost:33505/user/jenkins/test-data/e37915a4-8313-e328-91fb-37d541e8b214/WALs/1ab588ae992e,40419,1732433713579, archiveDir=hdfs://localhost:33505/user/jenkins/test-data/e37915a4-8313-e328-91fb-37d541e8b214/oldWALs, maxLogs=32 2024-11-24T07:35:15,268 DEBUG [RS:1;1ab588ae992e:40419 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/e37915a4-8313-e328-91fb-37d541e8b214/WALs/1ab588ae992e,40419,1732433713579/1ab588ae992e%2C40419%2C1732433713579.1732433715254, exclude list is [], retry=0 2024-11-24T07:35:15,268 DEBUG [RS:0;1ab588ae992e:46433 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/e37915a4-8313-e328-91fb-37d541e8b214/WALs/1ab588ae992e,46433,1732433713473/1ab588ae992e%2C46433%2C1732433713473.1732433715254, exclude list is [], retry=0 2024-11-24T07:35:15,269 DEBUG [RS:2;1ab588ae992e:40611 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/e37915a4-8313-e328-91fb-37d541e8b214/WALs/1ab588ae992e,40611,1732433713633/1ab588ae992e%2C40611%2C1732433713633.1732433715254, exclude list is [], retry=0 2024-11-24T07:35:15,273 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:42685,DS-467ee9a1-f3ca-4279-9b61-69a2a6a0308f,DISK] 2024-11-24T07:35:15,273 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:42685,DS-467ee9a1-f3ca-4279-9b61-69a2a6a0308f,DISK] 2024-11-24T07:35:15,273 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41971,DS-a67a12dc-5f72-4d11-ac5a-3e225e33c62a,DISK] 2024-11-24T07:35:15,273 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41971,DS-a67a12dc-5f72-4d11-ac5a-3e225e33c62a,DISK] 2024-11-24T07:35:15,273 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41161,DS-b0260eac-1380-4e84-bf22-a60109fdf5f6,DISK] 2024-11-24T07:35:15,274 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41161,DS-b0260eac-1380-4e84-bf22-a60109fdf5f6,DISK] 2024-11-24T07:35:15,274 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41161,DS-b0260eac-1380-4e84-bf22-a60109fdf5f6,DISK] 2024-11-24T07:35:15,274 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41971,DS-a67a12dc-5f72-4d11-ac5a-3e225e33c62a,DISK] 2024-11-24T07:35:15,274 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:42685,DS-467ee9a1-f3ca-4279-9b61-69a2a6a0308f,DISK] 2024-11-24T07:35:15,280 INFO [RS:0;1ab588ae992e:46433 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/e37915a4-8313-e328-91fb-37d541e8b214/WALs/1ab588ae992e,46433,1732433713473/1ab588ae992e%2C46433%2C1732433713473.1732433715254 2024-11-24T07:35:15,280 INFO [RS:1;1ab588ae992e:40419 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/e37915a4-8313-e328-91fb-37d541e8b214/WALs/1ab588ae992e,40419,1732433713579/1ab588ae992e%2C40419%2C1732433713579.1732433715254 2024-11-24T07:35:15,281 DEBUG [RS:1;1ab588ae992e:40419 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:37111:37111),(127.0.0.1/127.0.0.1:35537:35537),(127.0.0.1/127.0.0.1:40647:40647)] 2024-11-24T07:35:15,281 INFO [RS:2;1ab588ae992e:40611 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/e37915a4-8313-e328-91fb-37d541e8b214/WALs/1ab588ae992e,40611,1732433713633/1ab588ae992e%2C40611%2C1732433713633.1732433715254 2024-11-24T07:35:15,282 DEBUG [RS:2;1ab588ae992e:40611 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:35537:35537),(127.0.0.1/127.0.0.1:40647:40647),(127.0.0.1/127.0.0.1:37111:37111)] 2024-11-24T07:35:15,283 DEBUG [RS:0;1ab588ae992e:46433 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:35537:35537),(127.0.0.1/127.0.0.1:37111:37111),(127.0.0.1/127.0.0.1:40647:40647)] 2024-11-24T07:35:15,418 DEBUG [1ab588ae992e:37719 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=3, allServersCount=3 2024-11-24T07:35:15,426 DEBUG [1ab588ae992e:37719 {}] balancer.BalancerClusterState(204): Hosts are {1ab588ae992e=0} racks are {/default-rack=0} 2024-11-24T07:35:15,432 DEBUG [1ab588ae992e:37719 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-11-24T07:35:15,432 DEBUG [1ab588ae992e:37719 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-11-24T07:35:15,432 DEBUG [1ab588ae992e:37719 {}] balancer.BalancerClusterState(303): server 2 has 0 regions 2024-11-24T07:35:15,432 DEBUG [1ab588ae992e:37719 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-11-24T07:35:15,432 DEBUG [1ab588ae992e:37719 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-11-24T07:35:15,432 DEBUG [1ab588ae992e:37719 {}] balancer.BalancerClusterState(310): server 2 is on host 0 2024-11-24T07:35:15,432 INFO [1ab588ae992e:37719 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-11-24T07:35:15,432 INFO [1ab588ae992e:37719 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-11-24T07:35:15,432 INFO [1ab588ae992e:37719 {}] balancer.BalancerClusterState(321): server 2 is on rack 0 2024-11-24T07:35:15,432 DEBUG [1ab588ae992e:37719 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-11-24T07:35:15,438 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=1ab588ae992e,40611,1732433713633 2024-11-24T07:35:15,444 INFO [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 1ab588ae992e,40611,1732433713633, state=OPENING 2024-11-24T07:35:15,508 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it 2024-11-24T07:35:15,520 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40611-0x1016bc48eea0003, quorum=127.0.0.1:49244, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-24T07:35:15,520 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37719-0x1016bc48eea0000, quorum=127.0.0.1:49244, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-24T07:35:15,520 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46433-0x1016bc48eea0001, quorum=127.0.0.1:49244, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-24T07:35:15,520 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40419-0x1016bc48eea0002, quorum=127.0.0.1:49244, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-24T07:35:15,521 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-24T07:35:15,521 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-24T07:35:15,521 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-24T07:35:15,521 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-24T07:35:15,522 DEBUG [PEWorker-3 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-24T07:35:15,524 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=1ab588ae992e,40611,1732433713633}] 2024-11-24T07:35:15,705 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-24T07:35:15,706 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:59343, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-24T07:35:15,719 INFO [RS_OPEN_META-regionserver/1ab588ae992e:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-11-24T07:35:15,719 INFO [RS_OPEN_META-regionserver/1ab588ae992e:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-24T07:35:15,720 INFO [RS_OPEN_META-regionserver/1ab588ae992e:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-11-24T07:35:15,724 INFO [RS_OPEN_META-regionserver/1ab588ae992e:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=1ab588ae992e%2C40611%2C1732433713633.meta, suffix=.meta, logDir=hdfs://localhost:33505/user/jenkins/test-data/e37915a4-8313-e328-91fb-37d541e8b214/WALs/1ab588ae992e,40611,1732433713633, archiveDir=hdfs://localhost:33505/user/jenkins/test-data/e37915a4-8313-e328-91fb-37d541e8b214/oldWALs, maxLogs=32 2024-11-24T07:35:15,740 DEBUG [RS_OPEN_META-regionserver/1ab588ae992e:0-0 {event_type=M_RS_OPEN_META, pid=3}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/e37915a4-8313-e328-91fb-37d541e8b214/WALs/1ab588ae992e,40611,1732433713633/1ab588ae992e%2C40611%2C1732433713633.meta.1732433715726.meta, exclude list is [], retry=0 2024-11-24T07:35:15,744 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41971,DS-a67a12dc-5f72-4d11-ac5a-3e225e33c62a,DISK] 2024-11-24T07:35:15,744 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41161,DS-b0260eac-1380-4e84-bf22-a60109fdf5f6,DISK] 2024-11-24T07:35:15,744 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:42685,DS-467ee9a1-f3ca-4279-9b61-69a2a6a0308f,DISK] 2024-11-24T07:35:15,747 INFO [RS_OPEN_META-regionserver/1ab588ae992e:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/e37915a4-8313-e328-91fb-37d541e8b214/WALs/1ab588ae992e,40611,1732433713633/1ab588ae992e%2C40611%2C1732433713633.meta.1732433715726.meta 2024-11-24T07:35:15,748 DEBUG [RS_OPEN_META-regionserver/1ab588ae992e:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:35537:35537),(127.0.0.1/127.0.0.1:40647:40647),(127.0.0.1/127.0.0.1:37111:37111)] 2024-11-24T07:35:15,748 DEBUG [RS_OPEN_META-regionserver/1ab588ae992e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-11-24T07:35:15,750 DEBUG [RS_OPEN_META-regionserver/1ab588ae992e:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-11-24T07:35:15,752 DEBUG [RS_OPEN_META-regionserver/1ab588ae992e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-11-24T07:35:15,756 INFO [RS_OPEN_META-regionserver/1ab588ae992e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-11-24T07:35:15,760 DEBUG [RS_OPEN_META-regionserver/1ab588ae992e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-11-24T07:35:15,761 DEBUG [RS_OPEN_META-regionserver/1ab588ae992e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T07:35:15,761 DEBUG [RS_OPEN_META-regionserver/1ab588ae992e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-11-24T07:35:15,761 DEBUG [RS_OPEN_META-regionserver/1ab588ae992e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-11-24T07:35:15,764 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-24T07:35:15,766 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-24T07:35:15,766 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T07:35:15,767 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-24T07:35:15,767 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-24T07:35:15,768 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-24T07:35:15,768 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T07:35:15,769 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-24T07:35:15,769 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-24T07:35:15,770 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-24T07:35:15,771 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T07:35:15,771 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-24T07:35:15,771 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-24T07:35:15,773 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-24T07:35:15,773 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T07:35:15,774 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-24T07:35:15,774 DEBUG [RS_OPEN_META-regionserver/1ab588ae992e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-24T07:35:15,775 DEBUG [RS_OPEN_META-regionserver/1ab588ae992e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33505/user/jenkins/test-data/e37915a4-8313-e328-91fb-37d541e8b214/data/hbase/meta/1588230740 2024-11-24T07:35:15,778 DEBUG [RS_OPEN_META-regionserver/1ab588ae992e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33505/user/jenkins/test-data/e37915a4-8313-e328-91fb-37d541e8b214/data/hbase/meta/1588230740 2024-11-24T07:35:15,780 DEBUG [RS_OPEN_META-regionserver/1ab588ae992e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-24T07:35:15,780 DEBUG [RS_OPEN_META-regionserver/1ab588ae992e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-24T07:35:15,781 DEBUG [RS_OPEN_META-regionserver/1ab588ae992e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-24T07:35:15,784 DEBUG [RS_OPEN_META-regionserver/1ab588ae992e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-24T07:35:15,785 INFO [RS_OPEN_META-regionserver/1ab588ae992e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=71745742, jitterRate=0.06909486651420593}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-24T07:35:15,785 DEBUG [RS_OPEN_META-regionserver/1ab588ae992e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-11-24T07:35:15,786 DEBUG [RS_OPEN_META-regionserver/1ab588ae992e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1732433715762Writing region info on filesystem at 1732433715762Initializing all the Stores at 1732433715764 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732433715764Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732433715764Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732433715764Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732433715764Cleaning up temporary data from old regions at 1732433715780 (+16 ms)Running coprocessor post-open hooks at 1732433715785 (+5 ms)Region opened successfully at 1732433715786 (+1 ms) 2024-11-24T07:35:15,794 INFO [RS_OPEN_META-regionserver/1ab588ae992e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1732433715696 2024-11-24T07:35:15,807 DEBUG [RS_OPEN_META-regionserver/1ab588ae992e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-11-24T07:35:15,807 INFO [RS_OPEN_META-regionserver/1ab588ae992e:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-11-24T07:35:15,809 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=1ab588ae992e,40611,1732433713633 2024-11-24T07:35:15,811 INFO [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 1ab588ae992e,40611,1732433713633, state=OPEN 2024-11-24T07:35:15,856 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40611-0x1016bc48eea0003, quorum=127.0.0.1:49244, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-24T07:35:15,856 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46433-0x1016bc48eea0001, quorum=127.0.0.1:49244, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-24T07:35:15,856 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40419-0x1016bc48eea0002, quorum=127.0.0.1:49244, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-24T07:35:15,857 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37719-0x1016bc48eea0000, quorum=127.0.0.1:49244, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-24T07:35:15,857 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-24T07:35:15,857 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-24T07:35:15,857 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-24T07:35:15,857 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-24T07:35:15,857 DEBUG [PEWorker-5 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=1ab588ae992e,40611,1732433713633 2024-11-24T07:35:15,865 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2 2024-11-24T07:35:15,865 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=1ab588ae992e,40611,1732433713633 in 333 msec 2024-11-24T07:35:15,874 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1 2024-11-24T07:35:15,874 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 863 msec 2024-11-24T07:35:15,875 DEBUG [PEWorker-2 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-24T07:35:15,875 INFO [PEWorker-2 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces 2024-11-24T07:35:15,916 DEBUG [PEWorker-2 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-24T07:35:15,917 DEBUG [PEWorker-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=1ab588ae992e,40611,1732433713633, seqNum=-1] 2024-11-24T07:35:15,939 DEBUG [PEWorker-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-24T07:35:15,942 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:49943, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-24T07:35:15,964 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 1.2280 sec 2024-11-24T07:35:15,964 INFO [master/1ab588ae992e:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1732433715964, completionTime=-1 2024-11-24T07:35:15,966 INFO [master/1ab588ae992e:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=3; waited=0ms, expected min=3 server(s), max=3 server(s), master is running 2024-11-24T07:35:15,967 DEBUG [master/1ab588ae992e:0:becomeActiveMaster {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-24T07:35:15,989 INFO [master/1ab588ae992e:0:becomeActiveMaster {}] assignment.AssignmentManager(1776): Number of RegionServers=3 2024-11-24T07:35:15,990 INFO [master/1ab588ae992e:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732433775990 2024-11-24T07:35:15,990 INFO [master/1ab588ae992e:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732433835990 2024-11-24T07:35:15,990 INFO [master/1ab588ae992e:0:becomeActiveMaster {}] assignment.AssignmentManager(1783): Joined the cluster in 23 msec 2024-11-24T07:35:15,991 DEBUG [master/1ab588ae992e:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 1588230740 changed from -1.0 to 0.0, refreshing cache 2024-11-24T07:35:15,998 INFO [master/1ab588ae992e:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=1ab588ae992e,37719,1732433712713-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-24T07:35:15,998 INFO [master/1ab588ae992e:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=1ab588ae992e,37719,1732433712713-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-24T07:35:15,998 INFO [master/1ab588ae992e:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=1ab588ae992e,37719,1732433712713-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-24T07:35:16,000 INFO [master/1ab588ae992e:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-1ab588ae992e:37719, period=300000, unit=MILLISECONDS is enabled. 2024-11-24T07:35:16,000 INFO [master/1ab588ae992e:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2024-11-24T07:35:16,001 INFO [master/1ab588ae992e:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled. 2024-11-24T07:35:16,006 DEBUG [master/1ab588ae992e:0.Chore.1 {}] janitor.CatalogJanitor(180): 2024-11-24T07:35:16,026 INFO [master/1ab588ae992e:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 2.284sec 2024-11-24T07:35:16,028 INFO [master/1ab588ae992e:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2024-11-24T07:35:16,030 INFO [master/1ab588ae992e:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2024-11-24T07:35:16,031 INFO [master/1ab588ae992e:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2024-11-24T07:35:16,031 INFO [master/1ab588ae992e:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2024-11-24T07:35:16,031 INFO [master/1ab588ae992e:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2024-11-24T07:35:16,032 INFO [master/1ab588ae992e:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=1ab588ae992e,37719,1732433712713-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-24T07:35:16,033 INFO [master/1ab588ae992e:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=1ab588ae992e,37719,1732433712713-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2024-11-24T07:35:16,037 DEBUG [master/1ab588ae992e:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds 2024-11-24T07:35:16,038 INFO [master/1ab588ae992e:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2024-11-24T07:35:16,038 INFO [master/1ab588ae992e:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=1ab588ae992e,37719,1732433712713-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-24T07:35:16,083 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@ddf6023, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T07:35:16,084 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 1ab588ae992e,37719,-1 for getting cluster id 2024-11-24T07:35:16,087 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-24T07:35:16,096 DEBUG [HMaster-EventLoopGroup-2-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'c8d56efe-e808-4116-9fd2-fabfbc66fe32' 2024-11-24T07:35:16,099 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-24T07:35:16,099 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "c8d56efe-e808-4116-9fd2-fabfbc66fe32" 2024-11-24T07:35:16,100 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@71d18408, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T07:35:16,100 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [1ab588ae992e,37719,-1] 2024-11-24T07:35:16,102 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-24T07:35:16,104 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T07:35:16,105 INFO [HMaster-EventLoopGroup-2-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:55078, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-24T07:35:16,108 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@65c83e31, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T07:35:16,108 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-24T07:35:16,115 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=1ab588ae992e,40611,1732433713633, seqNum=-1] 2024-11-24T07:35:16,116 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-24T07:35:16,118 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:38364, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-24T07:35:16,141 INFO [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=1ab588ae992e,37719,1732433712713 2024-11-24T07:35:16,142 INFO [Time-limited test {}] wal.AbstractTestWALReplay(147): hbase.rootdir=hdfs://localhost:33505/hbase 2024-11-24T07:35:16,154 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestAsyncWALReplayValueCompression#testNameConflictWhenSplit0 Thread=364, OpenFileDescriptor=607, MaxFileDescriptor=1048576, SystemLoadAverage=269, ProcessCount=11, AvailableMemoryMB=2398 2024-11-24T07:35:16,177 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-24T07:35:16,181 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-24T07:35:16,183 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-24T07:35:16,188 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-38279809, suffix=, logDir=hdfs://localhost:33505/hbase/WALs/hregion-38279809, archiveDir=hdfs://localhost:33505/hbase/oldWALs, maxLogs=32 2024-11-24T07:35:16,201 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-38279809/hregion-38279809.1732433716189, exclude list is [], retry=0 2024-11-24T07:35:16,206 DEBUG [AsyncFSWAL-8-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41971,DS-a67a12dc-5f72-4d11-ac5a-3e225e33c62a,DISK] 2024-11-24T07:35:16,206 DEBUG [AsyncFSWAL-8-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41161,DS-b0260eac-1380-4e84-bf22-a60109fdf5f6,DISK] 2024-11-24T07:35:16,206 DEBUG [AsyncFSWAL-8-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:42685,DS-467ee9a1-f3ca-4279-9b61-69a2a6a0308f,DISK] 2024-11-24T07:35:16,210 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-38279809/hregion-38279809.1732433716189 2024-11-24T07:35:16,210 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:37111:37111),(127.0.0.1/127.0.0.1:40647:40647),(127.0.0.1/127.0.0.1:35537:35537)] 2024-11-24T07:35:16,210 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => da494ea642b481253342dedf4094abbb, NAME => 'testReplayEditsWrittenIntoWAL,,1732433716179.da494ea642b481253342dedf4094abbb.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testReplayEditsWrittenIntoWAL', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:33505/hbase 2024-11-24T07:35:16,222 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41161 is added to blk_1073741838_1014 (size=64) 2024-11-24T07:35:16,222 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41971 is added to blk_1073741838_1014 (size=64) 2024-11-24T07:35:16,223 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42685 is added to blk_1073741838_1014 (size=64) 2024-11-24T07:35:16,224 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenIntoWAL,,1732433716179.da494ea642b481253342dedf4094abbb.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T07:35:16,226 INFO [StoreOpener-da494ea642b481253342dedf4094abbb-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region da494ea642b481253342dedf4094abbb 2024-11-24T07:35:16,228 INFO [StoreOpener-da494ea642b481253342dedf4094abbb-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region da494ea642b481253342dedf4094abbb columnFamilyName a 2024-11-24T07:35:16,228 DEBUG [StoreOpener-da494ea642b481253342dedf4094abbb-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T07:35:16,229 INFO [StoreOpener-da494ea642b481253342dedf4094abbb-1 {}] regionserver.HStore(327): Store=da494ea642b481253342dedf4094abbb/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T07:35:16,230 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for da494ea642b481253342dedf4094abbb 2024-11-24T07:35:16,231 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33505/hbase/data/default/testReplayEditsWrittenIntoWAL/da494ea642b481253342dedf4094abbb 2024-11-24T07:35:16,232 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33505/hbase/data/default/testReplayEditsWrittenIntoWAL/da494ea642b481253342dedf4094abbb 2024-11-24T07:35:16,232 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for da494ea642b481253342dedf4094abbb 2024-11-24T07:35:16,232 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for da494ea642b481253342dedf4094abbb 2024-11-24T07:35:16,235 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for da494ea642b481253342dedf4094abbb 2024-11-24T07:35:16,239 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:33505/hbase/data/default/testReplayEditsWrittenIntoWAL/da494ea642b481253342dedf4094abbb/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-24T07:35:16,240 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened da494ea642b481253342dedf4094abbb; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=74179791, jitterRate=0.10536502301692963}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-24T07:35:16,242 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for da494ea642b481253342dedf4094abbb: Writing region info on filesystem at 1732433716224Initializing all the Stores at 1732433716225 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732433716226 (+1 ms)Cleaning up temporary data from old regions at 1732433716232 (+6 ms)Region opened successfully at 1732433716242 (+10 ms) 2024-11-24T07:35:16,242 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing da494ea642b481253342dedf4094abbb, disabling compactions & flushes 2024-11-24T07:35:16,242 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsWrittenIntoWAL,,1732433716179.da494ea642b481253342dedf4094abbb. 2024-11-24T07:35:16,242 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsWrittenIntoWAL,,1732433716179.da494ea642b481253342dedf4094abbb. 2024-11-24T07:35:16,242 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsWrittenIntoWAL,,1732433716179.da494ea642b481253342dedf4094abbb. after waiting 0 ms 2024-11-24T07:35:16,242 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsWrittenIntoWAL,,1732433716179.da494ea642b481253342dedf4094abbb. 2024-11-24T07:35:16,243 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsWrittenIntoWAL,,1732433716179.da494ea642b481253342dedf4094abbb. 2024-11-24T07:35:16,243 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for da494ea642b481253342dedf4094abbb: Waiting for close lock at 1732433716242Disabling compacts and flushes for region at 1732433716242Disabling writes for close at 1732433716242Writing region close event to WAL at 1732433716243 (+1 ms)Closed at 1732433716243 2024-11-24T07:35:16,248 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41971 is added to blk_1073741837_1013 (size=95) 2024-11-24T07:35:16,248 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42685 is added to blk_1073741837_1013 (size=95) 2024-11-24T07:35:16,248 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41161 is added to blk_1073741837_1013 (size=95) 2024-11-24T07:35:16,254 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-11-24T07:35:16,255 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-38279809:(num 1732433716189) 2024-11-24T07:35:16,257 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor testMonitor 2024-11-24T07:35:16,274 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42685 is added to blk_1073741839_1015 (size=320) 2024-11-24T07:35:16,274 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41161 is added to blk_1073741839_1015 (size=320) 2024-11-24T07:35:16,274 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41971 is added to blk_1073741839_1015 (size=320) 2024-11-24T07:35:16,276 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor testMonitor 2024-11-24T07:35:16,287 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41971 is added to blk_1073741840_1016 (size=253) 2024-11-24T07:35:16,288 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42685 is added to blk_1073741840_1016 (size=253) 2024-11-24T07:35:16,288 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41161 is added to blk_1073741840_1016 (size=253) 2024-11-24T07:35:16,307 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:33505/hbase/WALs/testnameconflictwhensplit0-manual,16010,1732433716176/wal-1, size=320 (320bytes) 2024-11-24T07:35:16,308 DEBUG [Time-limited test {}] util.RecoverLeaseFSUtils(47): Initialize RecoverLeaseFSUtils 2024-11-24T07:35:16,308 DEBUG [Time-limited test {}] util.RecoverLeaseFSUtils(59): set recoverLeaseMethod to org.apache.hadoop.fs.LeaseRecoverable.recoverLease() 2024-11-24T07:35:16,308 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:33505/hbase/WALs/testnameconflictwhensplit0-manual,16010,1732433716176/wal-1 2024-11-24T07:35:16,313 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:33505/hbase/WALs/testnameconflictwhensplit0-manual,16010,1732433716176/wal-1 after 3ms 2024-11-24T07:35:16,318 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:33505/hbase/WALs/testnameconflictwhensplit0-manual,16010,1732433716176/wal-1: isRecoveredEdits=false, hasTagCompression=true, hasValueCompression=true, valueCompressionType=GZ 2024-11-24T07:35:16,331 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:33505/hbase/WALs/testnameconflictwhensplit0-manual,16010,1732433716176/wal-1 took 25ms 2024-11-24T07:35:16,344 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:33505/hbase/WALs/testnameconflictwhensplit0-manual,16010,1732433716176/wal-1 so closing down 2024-11-24T07:35:16,345 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-24T07:35:16,347 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000001-wal-1.temp 2024-11-24T07:35:16,349 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:33505/hbase/data/default/testReplayEditsWrittenIntoWAL/da494ea642b481253342dedf4094abbb/recovered.edits/0000000000000000001-wal-1.temp 2024-11-24T07:35:16,350 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-11-24T07:35:16,359 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41161 is added to blk_1073741841_1017 (size=320) 2024-11-24T07:35:16,359 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42685 is added to blk_1073741841_1017 (size=320) 2024-11-24T07:35:16,360 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41971 is added to blk_1073741841_1017 (size=320) 2024-11-24T07:35:16,363 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:33505/hbase/data/default/testReplayEditsWrittenIntoWAL/da494ea642b481253342dedf4094abbb/recovered.edits/0000000000000000001-wal-1.temp (wrote 2 edits, skipped 0 edits in 0 ms) 2024-11-24T07:35:16,366 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:33505/hbase/data/default/testReplayEditsWrittenIntoWAL/da494ea642b481253342dedf4094abbb/recovered.edits/0000000000000000001-wal-1.temp to hdfs://localhost:33505/hbase/data/default/testReplayEditsWrittenIntoWAL/da494ea642b481253342dedf4094abbb/recovered.edits/0000000000000000002 2024-11-24T07:35:16,369 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 2 edits across 1 Regions in 34 ms; skipped=0; WAL=hdfs://localhost:33505/hbase/WALs/testnameconflictwhensplit0-manual,16010,1732433716176/wal-1, size=320, length=320, corrupted=false, cancelled=false 2024-11-24T07:35:16,369 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:33505/hbase/WALs/testnameconflictwhensplit0-manual,16010,1732433716176/wal-1, journal: Splitting hdfs://localhost:33505/hbase/WALs/testnameconflictwhensplit0-manual,16010,1732433716176/wal-1, size=320 (320bytes) at 1732433716308Finishing writing output for hdfs://localhost:33505/hbase/WALs/testnameconflictwhensplit0-manual,16010,1732433716176/wal-1 so closing down at 1732433716344 (+36 ms)Creating recovered edits writer path=hdfs://localhost:33505/hbase/data/default/testReplayEditsWrittenIntoWAL/da494ea642b481253342dedf4094abbb/recovered.edits/0000000000000000001-wal-1.temp at 1732433716349 (+5 ms)3 split writer threads finished at 1732433716350 (+1 ms)Closed recovered edits writer path=hdfs://localhost:33505/hbase/data/default/testReplayEditsWrittenIntoWAL/da494ea642b481253342dedf4094abbb/recovered.edits/0000000000000000001-wal-1.temp (wrote 2 edits, skipped 0 edits in 0 ms) at 1732433716363 (+13 ms)Rename recovered edits hdfs://localhost:33505/hbase/data/default/testReplayEditsWrittenIntoWAL/da494ea642b481253342dedf4094abbb/recovered.edits/0000000000000000001-wal-1.temp to hdfs://localhost:33505/hbase/data/default/testReplayEditsWrittenIntoWAL/da494ea642b481253342dedf4094abbb/recovered.edits/0000000000000000002 at 1732433716366 (+3 ms)Processed 2 edits across 1 Regions in 34 ms; skipped=0; WAL=hdfs://localhost:33505/hbase/WALs/testnameconflictwhensplit0-manual,16010,1732433716176/wal-1, size=320, length=320, corrupted=false, cancelled=false at 1732433716369 (+3 ms) 2024-11-24T07:35:16,384 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:33505/hbase/WALs/testnameconflictwhensplit0-manual,16010,1732433716176/wal-2, size=253 (253bytes) 2024-11-24T07:35:16,384 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:33505/hbase/WALs/testnameconflictwhensplit0-manual,16010,1732433716176/wal-2 2024-11-24T07:35:16,385 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:33505/hbase/WALs/testnameconflictwhensplit0-manual,16010,1732433716176/wal-2 after 0ms 2024-11-24T07:35:16,389 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:33505/hbase/WALs/testnameconflictwhensplit0-manual,16010,1732433716176/wal-2: isRecoveredEdits=false, hasTagCompression=true, hasValueCompression=true, valueCompressionType=GZ 2024-11-24T07:35:16,389 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:33505/hbase/WALs/testnameconflictwhensplit0-manual,16010,1732433716176/wal-2 took 6ms 2024-11-24T07:35:16,392 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:33505/hbase/WALs/testnameconflictwhensplit0-manual,16010,1732433716176/wal-2 so closing down 2024-11-24T07:35:16,392 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-24T07:35:16,394 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000002-wal-2.temp 2024-11-24T07:35:16,396 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:33505/hbase/data/default/testReplayEditsWrittenIntoWAL/da494ea642b481253342dedf4094abbb/recovered.edits/0000000000000000002-wal-2.temp 2024-11-24T07:35:16,396 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-11-24T07:35:16,404 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41971 is added to blk_1073741842_1018 (size=253) 2024-11-24T07:35:16,404 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41161 is added to blk_1073741842_1018 (size=253) 2024-11-24T07:35:16,404 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42685 is added to blk_1073741842_1018 (size=253) 2024-11-24T07:35:16,405 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:33505/hbase/data/default/testReplayEditsWrittenIntoWAL/da494ea642b481253342dedf4094abbb/recovered.edits/0000000000000000002-wal-2.temp (wrote 1 edits, skipped 0 edits in 0 ms) 2024-11-24T07:35:16,410 DEBUG [split-log-closeStream-pool-0 {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:33505/hbase/data/default/testReplayEditsWrittenIntoWAL/da494ea642b481253342dedf4094abbb/recovered.edits/0000000000000000002: isRecoveredEdits=true, hasTagCompression=true, hasValueCompression=true, valueCompressionType=GZ 2024-11-24T07:35:16,412 WARN [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(184): Found existing old edits file and we have less entries. Deleting hdfs://localhost:33505/hbase/data/default/testReplayEditsWrittenIntoWAL/da494ea642b481253342dedf4094abbb/recovered.edits/0000000000000000002-wal-2.temp, length=253 2024-11-24T07:35:16,414 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 1 edits across 1 Regions in 25 ms; skipped=0; WAL=hdfs://localhost:33505/hbase/WALs/testnameconflictwhensplit0-manual,16010,1732433716176/wal-2, size=253, length=253, corrupted=false, cancelled=false 2024-11-24T07:35:16,414 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:33505/hbase/WALs/testnameconflictwhensplit0-manual,16010,1732433716176/wal-2, journal: Splitting hdfs://localhost:33505/hbase/WALs/testnameconflictwhensplit0-manual,16010,1732433716176/wal-2, size=253 (253bytes) at 1732433716384Finishing writing output for hdfs://localhost:33505/hbase/WALs/testnameconflictwhensplit0-manual,16010,1732433716176/wal-2 so closing down at 1732433716392 (+8 ms)Creating recovered edits writer path=hdfs://localhost:33505/hbase/data/default/testReplayEditsWrittenIntoWAL/da494ea642b481253342dedf4094abbb/recovered.edits/0000000000000000002-wal-2.temp at 1732433716396 (+4 ms)3 split writer threads finished at 1732433716396Closed recovered edits writer path=hdfs://localhost:33505/hbase/data/default/testReplayEditsWrittenIntoWAL/da494ea642b481253342dedf4094abbb/recovered.edits/0000000000000000002-wal-2.temp (wrote 1 edits, skipped 0 edits in 0 ms) at 1732433716405 (+9 ms)Processed 1 edits across 1 Regions in 25 ms; skipped=0; WAL=hdfs://localhost:33505/hbase/WALs/testnameconflictwhensplit0-manual,16010,1732433716176/wal-2, size=253, length=253, corrupted=false, cancelled=false at 1732433716414 (+9 ms) 2024-11-24T07:35:16,414 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-24T07:35:16,417 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:33505/hbase/WALs/testnameconflictwhensplit0-manual,16010,1732433716176, archiveDir=hdfs://localhost:33505/hbase/oldWALs, maxLogs=32 2024-11-24T07:35:16,431 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testnameconflictwhensplit0-manual,16010,1732433716176/wal.1732433716419, exclude list is [], retry=0 2024-11-24T07:35:16,435 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41971,DS-a67a12dc-5f72-4d11-ac5a-3e225e33c62a,DISK] 2024-11-24T07:35:16,436 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41161,DS-b0260eac-1380-4e84-bf22-a60109fdf5f6,DISK] 2024-11-24T07:35:16,436 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:42685,DS-467ee9a1-f3ca-4279-9b61-69a2a6a0308f,DISK] 2024-11-24T07:35:16,439 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testnameconflictwhensplit0-manual,16010,1732433716176/wal.1732433716419 2024-11-24T07:35:16,439 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:37111:37111),(127.0.0.1/127.0.0.1:40647:40647),(127.0.0.1/127.0.0.1:35537:35537)] 2024-11-24T07:35:16,440 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => da494ea642b481253342dedf4094abbb, NAME => 'testReplayEditsWrittenIntoWAL,,1732433716179.da494ea642b481253342dedf4094abbb.', STARTKEY => '', ENDKEY => ''} 2024-11-24T07:35:16,440 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenIntoWAL,,1732433716179.da494ea642b481253342dedf4094abbb.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T07:35:16,440 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for da494ea642b481253342dedf4094abbb 2024-11-24T07:35:16,440 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for da494ea642b481253342dedf4094abbb 2024-11-24T07:35:16,443 INFO [StoreOpener-da494ea642b481253342dedf4094abbb-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region da494ea642b481253342dedf4094abbb 2024-11-24T07:35:16,444 INFO [StoreOpener-da494ea642b481253342dedf4094abbb-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region da494ea642b481253342dedf4094abbb columnFamilyName a 2024-11-24T07:35:16,445 DEBUG [StoreOpener-da494ea642b481253342dedf4094abbb-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T07:35:16,446 INFO [StoreOpener-da494ea642b481253342dedf4094abbb-1 {}] regionserver.HStore(327): Store=da494ea642b481253342dedf4094abbb/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T07:35:16,446 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for da494ea642b481253342dedf4094abbb 2024-11-24T07:35:16,447 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33505/hbase/data/default/testReplayEditsWrittenIntoWAL/da494ea642b481253342dedf4094abbb 2024-11-24T07:35:16,451 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:33505/hbase/data/default/testReplayEditsWrittenIntoWAL/da494ea642b481253342dedf4094abbb 2024-11-24T07:35:16,452 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:33505/hbase/data/default/testReplayEditsWrittenIntoWAL/da494ea642b481253342dedf4094abbb/recovered.edits/0000000000000000002 2024-11-24T07:35:16,456 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:33505/hbase/data/default/testReplayEditsWrittenIntoWAL/da494ea642b481253342dedf4094abbb/recovered.edits/0000000000000000002: isRecoveredEdits=true, hasTagCompression=true, hasValueCompression=true, valueCompressionType=GZ 2024-11-24T07:35:16,462 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 2, skipped 0, firstSequenceIdInLog=1, maxSequenceIdInLog=2, path=hdfs://localhost:33505/hbase/data/default/testReplayEditsWrittenIntoWAL/da494ea642b481253342dedf4094abbb/recovered.edits/0000000000000000002 2024-11-24T07:35:16,465 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing da494ea642b481253342dedf4094abbb 1/1 column families, dataSize=108 B heapSize=512 B 2024-11-24T07:35:16,514 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33505/hbase/data/default/testReplayEditsWrittenIntoWAL/da494ea642b481253342dedf4094abbb/.tmp/a/15e5d0cf6dc146ab8f6d9f0e63bba837 is 58, key is testReplayEditsWrittenIntoWAL/a:1/1732433716255/Put/seqid=0 2024-11-24T07:35:16,526 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41161 is added to blk_1073741844_1020 (size=5170) 2024-11-24T07:35:16,526 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41971 is added to blk_1073741844_1020 (size=5170) 2024-11-24T07:35:16,526 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42685 is added to blk_1073741844_1020 (size=5170) 2024-11-24T07:35:16,527 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=108 B at sequenceid=2 (bloomFilter=true), to=hdfs://localhost:33505/hbase/data/default/testReplayEditsWrittenIntoWAL/da494ea642b481253342dedf4094abbb/.tmp/a/15e5d0cf6dc146ab8f6d9f0e63bba837 2024-11-24T07:35:16,571 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33505/hbase/data/default/testReplayEditsWrittenIntoWAL/da494ea642b481253342dedf4094abbb/.tmp/a/15e5d0cf6dc146ab8f6d9f0e63bba837 as hdfs://localhost:33505/hbase/data/default/testReplayEditsWrittenIntoWAL/da494ea642b481253342dedf4094abbb/a/15e5d0cf6dc146ab8f6d9f0e63bba837 2024-11-24T07:35:16,582 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33505/hbase/data/default/testReplayEditsWrittenIntoWAL/da494ea642b481253342dedf4094abbb/a/15e5d0cf6dc146ab8f6d9f0e63bba837, entries=2, sequenceid=2, filesize=5.0 K 2024-11-24T07:35:16,588 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~108 B/108, heapSize ~496 B/496, currentSize=0 B/0 for da494ea642b481253342dedf4094abbb in 120ms, sequenceid=2, compaction requested=false; wal=null 2024-11-24T07:35:16,590 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:33505/hbase/data/default/testReplayEditsWrittenIntoWAL/da494ea642b481253342dedf4094abbb/recovered.edits/0000000000000000002 2024-11-24T07:35:16,591 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for da494ea642b481253342dedf4094abbb 2024-11-24T07:35:16,591 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for da494ea642b481253342dedf4094abbb 2024-11-24T07:35:16,594 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for da494ea642b481253342dedf4094abbb 2024-11-24T07:35:16,597 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:33505/hbase/data/default/testReplayEditsWrittenIntoWAL/da494ea642b481253342dedf4094abbb/recovered.edits/2.seqid, newMaxSeqId=2, maxSeqId=1 2024-11-24T07:35:16,599 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened da494ea642b481253342dedf4094abbb; next sequenceid=3; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=74916317, jitterRate=0.11634011566638947}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-24T07:35:16,600 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for da494ea642b481253342dedf4094abbb: Writing region info on filesystem at 1732433716440Initializing all the Stores at 1732433716442 (+2 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732433716442Obtaining lock to block concurrent updates at 1732433716465 (+23 ms)Preparing flush snapshotting stores in da494ea642b481253342dedf4094abbb at 1732433716465Finished memstore snapshotting testReplayEditsWrittenIntoWAL,,1732433716179.da494ea642b481253342dedf4094abbb., syncing WAL and waiting on mvcc, flushsize=dataSize=108, getHeapSize=496, getOffHeapSize=0, getCellsCount=2 at 1732433716468 (+3 ms)Flushing stores of testReplayEditsWrittenIntoWAL,,1732433716179.da494ea642b481253342dedf4094abbb. at 1732433716468Flushing da494ea642b481253342dedf4094abbb/a: creating writer at 1732433716469 (+1 ms)Flushing da494ea642b481253342dedf4094abbb/a: appending metadata at 1732433716502 (+33 ms)Flushing da494ea642b481253342dedf4094abbb/a: closing flushed file at 1732433716504 (+2 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@7981c500: reopening flushed file at 1732433716569 (+65 ms)Finished flush of dataSize ~108 B/108, heapSize ~496 B/496, currentSize=0 B/0 for da494ea642b481253342dedf4094abbb in 120ms, sequenceid=2, compaction requested=false; wal=null at 1732433716588 (+19 ms)Cleaning up temporary data from old regions at 1732433716591 (+3 ms)Region opened successfully at 1732433716600 (+9 ms) 2024-11-24T07:35:16,625 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestAsyncWALReplayValueCompression#testNameConflictWhenSplit0 Thread=376 (was 364) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-166876350_22 at /127.0.0.1:55576 [Receiving block BP-1337235608-172.17.0.3-1732433708251:blk_1073741843_1019] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-1337235608-172.17.0.3-1732433708251:blk_1073741843_1019, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-166876350_22 at /127.0.0.1:55436 [Waiting for operation #12] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-8-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-1337235608-172.17.0.3-1732433708251:blk_1073741843_1019, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: TestAsyncWALReplay-pool-0 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:62) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:883) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-166876350_22 at /127.0.0.1:33412 [Waiting for operation #5] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-166876350_22 at /127.0.0.1:37274 [Receiving block BP-1337235608-172.17.0.3-1732433708251:blk_1073741843_1019] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-8-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-166876350_22 at /127.0.0.1:37092 [Waiting for operation #5] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-8-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-1337235608-172.17.0.3-1732433708251:blk_1073741843_1019, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-0-hdfs://localhost:33505/hbase-prefix:default java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-166876350_22 at /127.0.0.1:33414 [Receiving block BP-1337235608-172.17.0.3-1732433708251:blk_1073741843_1019] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=703 (was 607) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=269 (was 269), ProcessCount=11 (was 11), AvailableMemoryMB=2380 (was 2398) 2024-11-24T07:35:16,637 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestAsyncWALReplayValueCompression#testNameConflictWhenSplit1 Thread=376, OpenFileDescriptor=703, MaxFileDescriptor=1048576, SystemLoadAverage=269, ProcessCount=11, AvailableMemoryMB=2378 2024-11-24T07:35:16,658 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-24T07:35:16,661 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-24T07:35:16,662 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-24T07:35:16,666 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-68857373, suffix=, logDir=hdfs://localhost:33505/hbase/WALs/hregion-68857373, archiveDir=hdfs://localhost:33505/hbase/oldWALs, maxLogs=32 2024-11-24T07:35:16,685 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-68857373/hregion-68857373.1732433716668, exclude list is [], retry=0 2024-11-24T07:35:16,689 DEBUG [AsyncFSWAL-10-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:42685,DS-467ee9a1-f3ca-4279-9b61-69a2a6a0308f,DISK] 2024-11-24T07:35:16,689 DEBUG [AsyncFSWAL-10-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41971,DS-a67a12dc-5f72-4d11-ac5a-3e225e33c62a,DISK] 2024-11-24T07:35:16,690 DEBUG [AsyncFSWAL-10-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41161,DS-b0260eac-1380-4e84-bf22-a60109fdf5f6,DISK] 2024-11-24T07:35:16,693 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-68857373/hregion-68857373.1732433716668 2024-11-24T07:35:16,693 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:35537:35537),(127.0.0.1/127.0.0.1:37111:37111),(127.0.0.1/127.0.0.1:40647:40647)] 2024-11-24T07:35:16,693 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => c09918c1a3dc70aaa0bfcefbcc346305, NAME => 'testReplayEditsWrittenIntoWAL,,1732433716659.c09918c1a3dc70aaa0bfcefbcc346305.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testReplayEditsWrittenIntoWAL', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:33505/hbase 2024-11-24T07:35:16,709 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41971 is added to blk_1073741846_1022 (size=64) 2024-11-24T07:35:16,709 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42685 is added to blk_1073741846_1022 (size=64) 2024-11-24T07:35:16,710 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41161 is added to blk_1073741846_1022 (size=64) 2024-11-24T07:35:16,711 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenIntoWAL,,1732433716659.c09918c1a3dc70aaa0bfcefbcc346305.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T07:35:16,713 INFO [StoreOpener-c09918c1a3dc70aaa0bfcefbcc346305-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region c09918c1a3dc70aaa0bfcefbcc346305 2024-11-24T07:35:16,716 INFO [StoreOpener-c09918c1a3dc70aaa0bfcefbcc346305-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region c09918c1a3dc70aaa0bfcefbcc346305 columnFamilyName a 2024-11-24T07:35:16,716 DEBUG [StoreOpener-c09918c1a3dc70aaa0bfcefbcc346305-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T07:35:16,717 INFO [StoreOpener-c09918c1a3dc70aaa0bfcefbcc346305-1 {}] regionserver.HStore(327): Store=c09918c1a3dc70aaa0bfcefbcc346305/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T07:35:16,717 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for c09918c1a3dc70aaa0bfcefbcc346305 2024-11-24T07:35:16,718 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33505/hbase/data/default/testReplayEditsWrittenIntoWAL/c09918c1a3dc70aaa0bfcefbcc346305 2024-11-24T07:35:16,719 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33505/hbase/data/default/testReplayEditsWrittenIntoWAL/c09918c1a3dc70aaa0bfcefbcc346305 2024-11-24T07:35:16,719 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for c09918c1a3dc70aaa0bfcefbcc346305 2024-11-24T07:35:16,719 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for c09918c1a3dc70aaa0bfcefbcc346305 2024-11-24T07:35:16,722 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for c09918c1a3dc70aaa0bfcefbcc346305 2024-11-24T07:35:16,725 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:33505/hbase/data/default/testReplayEditsWrittenIntoWAL/c09918c1a3dc70aaa0bfcefbcc346305/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-24T07:35:16,725 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened c09918c1a3dc70aaa0bfcefbcc346305; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=61709701, jitterRate=-0.0804537981748581}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-24T07:35:16,726 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for c09918c1a3dc70aaa0bfcefbcc346305: Writing region info on filesystem at 1732433716711Initializing all the Stores at 1732433716712 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732433716712Cleaning up temporary data from old regions at 1732433716719 (+7 ms)Region opened successfully at 1732433716726 (+7 ms) 2024-11-24T07:35:16,726 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing c09918c1a3dc70aaa0bfcefbcc346305, disabling compactions & flushes 2024-11-24T07:35:16,726 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsWrittenIntoWAL,,1732433716659.c09918c1a3dc70aaa0bfcefbcc346305. 2024-11-24T07:35:16,726 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsWrittenIntoWAL,,1732433716659.c09918c1a3dc70aaa0bfcefbcc346305. 2024-11-24T07:35:16,726 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsWrittenIntoWAL,,1732433716659.c09918c1a3dc70aaa0bfcefbcc346305. after waiting 0 ms 2024-11-24T07:35:16,726 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsWrittenIntoWAL,,1732433716659.c09918c1a3dc70aaa0bfcefbcc346305. 2024-11-24T07:35:16,726 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsWrittenIntoWAL,,1732433716659.c09918c1a3dc70aaa0bfcefbcc346305. 2024-11-24T07:35:16,726 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for c09918c1a3dc70aaa0bfcefbcc346305: Waiting for close lock at 1732433716726Disabling compacts and flushes for region at 1732433716726Disabling writes for close at 1732433716726Writing region close event to WAL at 1732433716726Closed at 1732433716726 2024-11-24T07:35:16,730 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41161 is added to blk_1073741845_1021 (size=95) 2024-11-24T07:35:16,730 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42685 is added to blk_1073741845_1021 (size=95) 2024-11-24T07:35:16,730 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41971 is added to blk_1073741845_1021 (size=95) 2024-11-24T07:35:16,733 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-11-24T07:35:16,733 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-68857373:(num 1732433716668) 2024-11-24T07:35:16,735 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor testMonitor 2024-11-24T07:35:16,746 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42685 is added to blk_1073741847_1023 (size=320) 2024-11-24T07:35:16,747 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41161 is added to blk_1073741847_1023 (size=320) 2024-11-24T07:35:16,749 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor testMonitor 2024-11-24T07:35:16,753 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41971 is added to blk_1073741847_1023 (size=320) 2024-11-24T07:35:16,760 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41971 is added to blk_1073741848_1024 (size=253) 2024-11-24T07:35:16,760 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41161 is added to blk_1073741848_1024 (size=253) 2024-11-24T07:35:16,760 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42685 is added to blk_1073741848_1024 (size=253) 2024-11-24T07:35:16,777 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:33505/hbase/WALs/testnameconflictwhensplit1-manual,16010,1732433716657/wal-2, size=253 (253bytes) 2024-11-24T07:35:16,777 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:33505/hbase/WALs/testnameconflictwhensplit1-manual,16010,1732433716657/wal-2 2024-11-24T07:35:16,779 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:33505/hbase/WALs/testnameconflictwhensplit1-manual,16010,1732433716657/wal-2 after 1ms 2024-11-24T07:35:16,782 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:33505/hbase/WALs/testnameconflictwhensplit1-manual,16010,1732433716657/wal-2: isRecoveredEdits=false, hasTagCompression=true, hasValueCompression=true, valueCompressionType=GZ 2024-11-24T07:35:16,782 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:33505/hbase/WALs/testnameconflictwhensplit1-manual,16010,1732433716657/wal-2 took 5ms 2024-11-24T07:35:16,785 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:33505/hbase/WALs/testnameconflictwhensplit1-manual,16010,1732433716657/wal-2 so closing down 2024-11-24T07:35:16,785 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-24T07:35:16,787 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000002-wal-2.temp 2024-11-24T07:35:16,789 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:33505/hbase/data/default/testReplayEditsWrittenIntoWAL/c09918c1a3dc70aaa0bfcefbcc346305/recovered.edits/0000000000000000002-wal-2.temp 2024-11-24T07:35:16,789 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-11-24T07:35:16,797 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41971 is added to blk_1073741849_1025 (size=253) 2024-11-24T07:35:16,797 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42685 is added to blk_1073741849_1025 (size=253) 2024-11-24T07:35:16,797 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41161 is added to blk_1073741849_1025 (size=253) 2024-11-24T07:35:16,798 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:33505/hbase/data/default/testReplayEditsWrittenIntoWAL/c09918c1a3dc70aaa0bfcefbcc346305/recovered.edits/0000000000000000002-wal-2.temp (wrote 1 edits, skipped 0 edits in 0 ms) 2024-11-24T07:35:16,801 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:33505/hbase/data/default/testReplayEditsWrittenIntoWAL/c09918c1a3dc70aaa0bfcefbcc346305/recovered.edits/0000000000000000002-wal-2.temp to hdfs://localhost:33505/hbase/data/default/testReplayEditsWrittenIntoWAL/c09918c1a3dc70aaa0bfcefbcc346305/recovered.edits/0000000000000000002 2024-11-24T07:35:16,801 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 1 edits across 1 Regions in 18 ms; skipped=0; WAL=hdfs://localhost:33505/hbase/WALs/testnameconflictwhensplit1-manual,16010,1732433716657/wal-2, size=253, length=253, corrupted=false, cancelled=false 2024-11-24T07:35:16,801 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:33505/hbase/WALs/testnameconflictwhensplit1-manual,16010,1732433716657/wal-2, journal: Splitting hdfs://localhost:33505/hbase/WALs/testnameconflictwhensplit1-manual,16010,1732433716657/wal-2, size=253 (253bytes) at 1732433716777Finishing writing output for hdfs://localhost:33505/hbase/WALs/testnameconflictwhensplit1-manual,16010,1732433716657/wal-2 so closing down at 1732433716785 (+8 ms)Creating recovered edits writer path=hdfs://localhost:33505/hbase/data/default/testReplayEditsWrittenIntoWAL/c09918c1a3dc70aaa0bfcefbcc346305/recovered.edits/0000000000000000002-wal-2.temp at 1732433716789 (+4 ms)3 split writer threads finished at 1732433716789Closed recovered edits writer path=hdfs://localhost:33505/hbase/data/default/testReplayEditsWrittenIntoWAL/c09918c1a3dc70aaa0bfcefbcc346305/recovered.edits/0000000000000000002-wal-2.temp (wrote 1 edits, skipped 0 edits in 0 ms) at 1732433716798 (+9 ms)Rename recovered edits hdfs://localhost:33505/hbase/data/default/testReplayEditsWrittenIntoWAL/c09918c1a3dc70aaa0bfcefbcc346305/recovered.edits/0000000000000000002-wal-2.temp to hdfs://localhost:33505/hbase/data/default/testReplayEditsWrittenIntoWAL/c09918c1a3dc70aaa0bfcefbcc346305/recovered.edits/0000000000000000002 at 1732433716801 (+3 ms)Processed 1 edits across 1 Regions in 18 ms; skipped=0; WAL=hdfs://localhost:33505/hbase/WALs/testnameconflictwhensplit1-manual,16010,1732433716657/wal-2, size=253, length=253, corrupted=false, cancelled=false at 1732433716801 2024-11-24T07:35:16,815 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:33505/hbase/WALs/testnameconflictwhensplit1-manual,16010,1732433716657/wal-1, size=320 (320bytes) 2024-11-24T07:35:16,815 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:33505/hbase/WALs/testnameconflictwhensplit1-manual,16010,1732433716657/wal-1 2024-11-24T07:35:16,816 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:33505/hbase/WALs/testnameconflictwhensplit1-manual,16010,1732433716657/wal-1 after 1ms 2024-11-24T07:35:16,820 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:33505/hbase/WALs/testnameconflictwhensplit1-manual,16010,1732433716657/wal-1: isRecoveredEdits=false, hasTagCompression=true, hasValueCompression=true, valueCompressionType=GZ 2024-11-24T07:35:16,820 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:33505/hbase/WALs/testnameconflictwhensplit1-manual,16010,1732433716657/wal-1 took 5ms 2024-11-24T07:35:16,824 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:33505/hbase/WALs/testnameconflictwhensplit1-manual,16010,1732433716657/wal-1 so closing down 2024-11-24T07:35:16,824 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-24T07:35:16,826 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000001-wal-1.temp 2024-11-24T07:35:16,827 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:33505/hbase/data/default/testReplayEditsWrittenIntoWAL/c09918c1a3dc70aaa0bfcefbcc346305/recovered.edits/0000000000000000001-wal-1.temp 2024-11-24T07:35:16,828 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-11-24T07:35:16,837 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41971 is added to blk_1073741850_1026 (size=320) 2024-11-24T07:35:16,837 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41161 is added to blk_1073741850_1026 (size=320) 2024-11-24T07:35:16,838 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42685 is added to blk_1073741850_1026 (size=320) 2024-11-24T07:35:16,838 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:33505/hbase/data/default/testReplayEditsWrittenIntoWAL/c09918c1a3dc70aaa0bfcefbcc346305/recovered.edits/0000000000000000001-wal-1.temp (wrote 2 edits, skipped 0 edits in 0 ms) 2024-11-24T07:35:16,842 DEBUG [split-log-closeStream-pool-0 {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:33505/hbase/data/default/testReplayEditsWrittenIntoWAL/c09918c1a3dc70aaa0bfcefbcc346305/recovered.edits/0000000000000000002: isRecoveredEdits=true, hasTagCompression=true, hasValueCompression=true, valueCompressionType=GZ 2024-11-24T07:35:16,845 WARN [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(175): Found existing old edits file. It could be the result of a previous failed split attempt or we have duplicated wal entries. Deleting hdfs://localhost:33505/hbase/data/default/testReplayEditsWrittenIntoWAL/c09918c1a3dc70aaa0bfcefbcc346305/recovered.edits/0000000000000000002, length=253 2024-11-24T07:35:16,848 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:33505/hbase/data/default/testReplayEditsWrittenIntoWAL/c09918c1a3dc70aaa0bfcefbcc346305/recovered.edits/0000000000000000001-wal-1.temp to hdfs://localhost:33505/hbase/data/default/testReplayEditsWrittenIntoWAL/c09918c1a3dc70aaa0bfcefbcc346305/recovered.edits/0000000000000000002 2024-11-24T07:35:16,848 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 2 edits across 1 Regions in 27 ms; skipped=0; WAL=hdfs://localhost:33505/hbase/WALs/testnameconflictwhensplit1-manual,16010,1732433716657/wal-1, size=320, length=320, corrupted=false, cancelled=false 2024-11-24T07:35:16,848 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:33505/hbase/WALs/testnameconflictwhensplit1-manual,16010,1732433716657/wal-1, journal: Splitting hdfs://localhost:33505/hbase/WALs/testnameconflictwhensplit1-manual,16010,1732433716657/wal-1, size=320 (320bytes) at 1732433716815Finishing writing output for hdfs://localhost:33505/hbase/WALs/testnameconflictwhensplit1-manual,16010,1732433716657/wal-1 so closing down at 1732433716824 (+9 ms)Creating recovered edits writer path=hdfs://localhost:33505/hbase/data/default/testReplayEditsWrittenIntoWAL/c09918c1a3dc70aaa0bfcefbcc346305/recovered.edits/0000000000000000001-wal-1.temp at 1732433716827 (+3 ms)3 split writer threads finished at 1732433716828 (+1 ms)Closed recovered edits writer path=hdfs://localhost:33505/hbase/data/default/testReplayEditsWrittenIntoWAL/c09918c1a3dc70aaa0bfcefbcc346305/recovered.edits/0000000000000000001-wal-1.temp (wrote 2 edits, skipped 0 edits in 0 ms) at 1732433716838 (+10 ms)Rename recovered edits hdfs://localhost:33505/hbase/data/default/testReplayEditsWrittenIntoWAL/c09918c1a3dc70aaa0bfcefbcc346305/recovered.edits/0000000000000000001-wal-1.temp to hdfs://localhost:33505/hbase/data/default/testReplayEditsWrittenIntoWAL/c09918c1a3dc70aaa0bfcefbcc346305/recovered.edits/0000000000000000002 at 1732433716848 (+10 ms)Processed 2 edits across 1 Regions in 27 ms; skipped=0; WAL=hdfs://localhost:33505/hbase/WALs/testnameconflictwhensplit1-manual,16010,1732433716657/wal-1, size=320, length=320, corrupted=false, cancelled=false at 1732433716848 2024-11-24T07:35:16,849 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-24T07:35:16,852 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:33505/hbase/WALs/testnameconflictwhensplit1-manual,16010,1732433716657, archiveDir=hdfs://localhost:33505/hbase/oldWALs, maxLogs=32 2024-11-24T07:35:16,868 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testnameconflictwhensplit1-manual,16010,1732433716657/wal.1732433716853, exclude list is [], retry=0 2024-11-24T07:35:16,872 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41161,DS-b0260eac-1380-4e84-bf22-a60109fdf5f6,DISK] 2024-11-24T07:35:16,873 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:42685,DS-467ee9a1-f3ca-4279-9b61-69a2a6a0308f,DISK] 2024-11-24T07:35:16,873 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41971,DS-a67a12dc-5f72-4d11-ac5a-3e225e33c62a,DISK] 2024-11-24T07:35:16,876 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testnameconflictwhensplit1-manual,16010,1732433716657/wal.1732433716853 2024-11-24T07:35:16,876 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:40647:40647),(127.0.0.1/127.0.0.1:35537:35537),(127.0.0.1/127.0.0.1:37111:37111)] 2024-11-24T07:35:16,876 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => c09918c1a3dc70aaa0bfcefbcc346305, NAME => 'testReplayEditsWrittenIntoWAL,,1732433716659.c09918c1a3dc70aaa0bfcefbcc346305.', STARTKEY => '', ENDKEY => ''} 2024-11-24T07:35:16,876 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenIntoWAL,,1732433716659.c09918c1a3dc70aaa0bfcefbcc346305.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T07:35:16,876 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for c09918c1a3dc70aaa0bfcefbcc346305 2024-11-24T07:35:16,876 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for c09918c1a3dc70aaa0bfcefbcc346305 2024-11-24T07:35:16,878 INFO [StoreOpener-c09918c1a3dc70aaa0bfcefbcc346305-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region c09918c1a3dc70aaa0bfcefbcc346305 2024-11-24T07:35:16,881 INFO [StoreOpener-c09918c1a3dc70aaa0bfcefbcc346305-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region c09918c1a3dc70aaa0bfcefbcc346305 columnFamilyName a 2024-11-24T07:35:16,881 DEBUG [StoreOpener-c09918c1a3dc70aaa0bfcefbcc346305-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T07:35:16,882 INFO [StoreOpener-c09918c1a3dc70aaa0bfcefbcc346305-1 {}] regionserver.HStore(327): Store=c09918c1a3dc70aaa0bfcefbcc346305/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T07:35:16,882 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for c09918c1a3dc70aaa0bfcefbcc346305 2024-11-24T07:35:16,883 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33505/hbase/data/default/testReplayEditsWrittenIntoWAL/c09918c1a3dc70aaa0bfcefbcc346305 2024-11-24T07:35:16,886 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:33505/hbase/data/default/testReplayEditsWrittenIntoWAL/c09918c1a3dc70aaa0bfcefbcc346305 2024-11-24T07:35:16,887 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:33505/hbase/data/default/testReplayEditsWrittenIntoWAL/c09918c1a3dc70aaa0bfcefbcc346305/recovered.edits/0000000000000000002 2024-11-24T07:35:16,891 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:33505/hbase/data/default/testReplayEditsWrittenIntoWAL/c09918c1a3dc70aaa0bfcefbcc346305/recovered.edits/0000000000000000002: isRecoveredEdits=true, hasTagCompression=true, hasValueCompression=true, valueCompressionType=GZ 2024-11-24T07:35:16,893 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 2, skipped 0, firstSequenceIdInLog=1, maxSequenceIdInLog=2, path=hdfs://localhost:33505/hbase/data/default/testReplayEditsWrittenIntoWAL/c09918c1a3dc70aaa0bfcefbcc346305/recovered.edits/0000000000000000002 2024-11-24T07:35:16,893 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing c09918c1a3dc70aaa0bfcefbcc346305 1/1 column families, dataSize=108 B heapSize=512 B 2024-11-24T07:35:16,909 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33505/hbase/data/default/testReplayEditsWrittenIntoWAL/c09918c1a3dc70aaa0bfcefbcc346305/.tmp/a/9995419cee3b449db66e3b05794ddc7f is 58, key is testReplayEditsWrittenIntoWAL/a:1/1732433716734/Put/seqid=0 2024-11-24T07:35:16,918 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41161 is added to blk_1073741852_1028 (size=5170) 2024-11-24T07:35:16,918 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42685 is added to blk_1073741852_1028 (size=5170) 2024-11-24T07:35:16,919 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41971 is added to blk_1073741852_1028 (size=5170) 2024-11-24T07:35:16,919 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=108 B at sequenceid=2 (bloomFilter=true), to=hdfs://localhost:33505/hbase/data/default/testReplayEditsWrittenIntoWAL/c09918c1a3dc70aaa0bfcefbcc346305/.tmp/a/9995419cee3b449db66e3b05794ddc7f 2024-11-24T07:35:16,928 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33505/hbase/data/default/testReplayEditsWrittenIntoWAL/c09918c1a3dc70aaa0bfcefbcc346305/.tmp/a/9995419cee3b449db66e3b05794ddc7f as hdfs://localhost:33505/hbase/data/default/testReplayEditsWrittenIntoWAL/c09918c1a3dc70aaa0bfcefbcc346305/a/9995419cee3b449db66e3b05794ddc7f 2024-11-24T07:35:16,937 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33505/hbase/data/default/testReplayEditsWrittenIntoWAL/c09918c1a3dc70aaa0bfcefbcc346305/a/9995419cee3b449db66e3b05794ddc7f, entries=2, sequenceid=2, filesize=5.0 K 2024-11-24T07:35:16,938 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~108 B/108, heapSize ~496 B/496, currentSize=0 B/0 for c09918c1a3dc70aaa0bfcefbcc346305 in 45ms, sequenceid=2, compaction requested=false; wal=null 2024-11-24T07:35:16,939 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:33505/hbase/data/default/testReplayEditsWrittenIntoWAL/c09918c1a3dc70aaa0bfcefbcc346305/recovered.edits/0000000000000000002 2024-11-24T07:35:16,939 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for c09918c1a3dc70aaa0bfcefbcc346305 2024-11-24T07:35:16,939 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for c09918c1a3dc70aaa0bfcefbcc346305 2024-11-24T07:35:16,942 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for c09918c1a3dc70aaa0bfcefbcc346305 2024-11-24T07:35:16,945 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:33505/hbase/data/default/testReplayEditsWrittenIntoWAL/c09918c1a3dc70aaa0bfcefbcc346305/recovered.edits/2.seqid, newMaxSeqId=2, maxSeqId=1 2024-11-24T07:35:16,946 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened c09918c1a3dc70aaa0bfcefbcc346305; next sequenceid=3; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=63146018, jitterRate=-0.05905100703239441}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-24T07:35:16,946 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for c09918c1a3dc70aaa0bfcefbcc346305: Writing region info on filesystem at 1732433716876Initializing all the Stores at 1732433716878 (+2 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732433716878Obtaining lock to block concurrent updates at 1732433716893 (+15 ms)Preparing flush snapshotting stores in c09918c1a3dc70aaa0bfcefbcc346305 at 1732433716893Finished memstore snapshotting testReplayEditsWrittenIntoWAL,,1732433716659.c09918c1a3dc70aaa0bfcefbcc346305., syncing WAL and waiting on mvcc, flushsize=dataSize=108, getHeapSize=496, getOffHeapSize=0, getCellsCount=2 at 1732433716893Flushing stores of testReplayEditsWrittenIntoWAL,,1732433716659.c09918c1a3dc70aaa0bfcefbcc346305. at 1732433716893Flushing c09918c1a3dc70aaa0bfcefbcc346305/a: creating writer at 1732433716893Flushing c09918c1a3dc70aaa0bfcefbcc346305/a: appending metadata at 1732433716908 (+15 ms)Flushing c09918c1a3dc70aaa0bfcefbcc346305/a: closing flushed file at 1732433716908Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@5696c389: reopening flushed file at 1732433716927 (+19 ms)Finished flush of dataSize ~108 B/108, heapSize ~496 B/496, currentSize=0 B/0 for c09918c1a3dc70aaa0bfcefbcc346305 in 45ms, sequenceid=2, compaction requested=false; wal=null at 1732433716938 (+11 ms)Cleaning up temporary data from old regions at 1732433716939 (+1 ms)Region opened successfully at 1732433716946 (+7 ms) 2024-11-24T07:35:16,962 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestAsyncWALReplayValueCompression#testNameConflictWhenSplit1 Thread=386 (was 376) Potentially hanging thread: AsyncFSWAL-10-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-166876350_22 at /127.0.0.1:33412 [Waiting for operation #14] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-166876350_22 at /127.0.0.1:55650 [Receiving block BP-1337235608-172.17.0.3-1732433708251:blk_1073741851_1027] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-1337235608-172.17.0.3-1732433708251:blk_1073741851_1027, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-1337235608-172.17.0.3-1732433708251:blk_1073741851_1027, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-1337235608-172.17.0.3-1732433708251:blk_1073741851_1027, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-166876350_22 at /127.0.0.1:37092 [Waiting for operation #9] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-10-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-166876350_22 at /127.0.0.1:55436 [Waiting for operation #14] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-166876350_22 at /127.0.0.1:37374 [Receiving block BP-1337235608-172.17.0.3-1732433708251:blk_1073741851_1027] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-166876350_22 at /127.0.0.1:33476 [Receiving block BP-1337235608-172.17.0.3-1732433708251:blk_1073741851_1027] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-10-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=785 (was 703) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=269 (was 269), ProcessCount=11 (was 11), AvailableMemoryMB=2369 (was 2378) 2024-11-24T07:35:16,974 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestAsyncWALReplayValueCompression#testReplayEditsWrittenIntoWAL Thread=386, OpenFileDescriptor=785, MaxFileDescriptor=1048576, SystemLoadAverage=269, ProcessCount=11, AvailableMemoryMB=2369 2024-11-24T07:35:16,991 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-24T07:35:16,993 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-24T07:35:16,994 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-24T07:35:16,997 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-58985170, suffix=, logDir=hdfs://localhost:33505/hbase/WALs/hregion-58985170, archiveDir=hdfs://localhost:33505/hbase/oldWALs, maxLogs=32 2024-11-24T07:35:17,009 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-58985170/hregion-58985170.1732433716998, exclude list is [], retry=0 2024-11-24T07:35:17,012 DEBUG [AsyncFSWAL-12-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:42685,DS-467ee9a1-f3ca-4279-9b61-69a2a6a0308f,DISK] 2024-11-24T07:35:17,012 DEBUG [AsyncFSWAL-12-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41971,DS-a67a12dc-5f72-4d11-ac5a-3e225e33c62a,DISK] 2024-11-24T07:35:17,013 DEBUG [AsyncFSWAL-12-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41161,DS-b0260eac-1380-4e84-bf22-a60109fdf5f6,DISK] 2024-11-24T07:35:17,015 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-58985170/hregion-58985170.1732433716998 2024-11-24T07:35:17,015 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:35537:35537),(127.0.0.1/127.0.0.1:37111:37111),(127.0.0.1/127.0.0.1:40647:40647)] 2024-11-24T07:35:17,015 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => 0dafe4c4d07f4913c287e4eb7d43a7fe, NAME => 'testReplayEditsWrittenIntoWAL,,1732433716992.0dafe4c4d07f4913c287e4eb7d43a7fe.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testReplayEditsWrittenIntoWAL', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:33505/hbase 2024-11-24T07:35:17,025 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41161 is added to blk_1073741854_1030 (size=64) 2024-11-24T07:35:17,026 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41971 is added to blk_1073741854_1030 (size=64) 2024-11-24T07:35:17,026 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42685 is added to blk_1073741854_1030 (size=64) 2024-11-24T07:35:17,026 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenIntoWAL,,1732433716992.0dafe4c4d07f4913c287e4eb7d43a7fe.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T07:35:17,028 INFO [StoreOpener-0dafe4c4d07f4913c287e4eb7d43a7fe-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 0dafe4c4d07f4913c287e4eb7d43a7fe 2024-11-24T07:35:17,030 INFO [StoreOpener-0dafe4c4d07f4913c287e4eb7d43a7fe-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 0dafe4c4d07f4913c287e4eb7d43a7fe columnFamilyName a 2024-11-24T07:35:17,030 DEBUG [StoreOpener-0dafe4c4d07f4913c287e4eb7d43a7fe-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T07:35:17,030 INFO [StoreOpener-0dafe4c4d07f4913c287e4eb7d43a7fe-1 {}] regionserver.HStore(327): Store=0dafe4c4d07f4913c287e4eb7d43a7fe/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T07:35:17,031 INFO [StoreOpener-0dafe4c4d07f4913c287e4eb7d43a7fe-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 0dafe4c4d07f4913c287e4eb7d43a7fe 2024-11-24T07:35:17,032 INFO [StoreOpener-0dafe4c4d07f4913c287e4eb7d43a7fe-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 0dafe4c4d07f4913c287e4eb7d43a7fe columnFamilyName b 2024-11-24T07:35:17,032 DEBUG [StoreOpener-0dafe4c4d07f4913c287e4eb7d43a7fe-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T07:35:17,033 INFO [StoreOpener-0dafe4c4d07f4913c287e4eb7d43a7fe-1 {}] regionserver.HStore(327): Store=0dafe4c4d07f4913c287e4eb7d43a7fe/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T07:35:17,033 INFO [StoreOpener-0dafe4c4d07f4913c287e4eb7d43a7fe-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 0dafe4c4d07f4913c287e4eb7d43a7fe 2024-11-24T07:35:17,034 INFO [StoreOpener-0dafe4c4d07f4913c287e4eb7d43a7fe-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 0dafe4c4d07f4913c287e4eb7d43a7fe columnFamilyName c 2024-11-24T07:35:17,035 DEBUG [StoreOpener-0dafe4c4d07f4913c287e4eb7d43a7fe-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T07:35:17,035 INFO [StoreOpener-0dafe4c4d07f4913c287e4eb7d43a7fe-1 {}] regionserver.HStore(327): Store=0dafe4c4d07f4913c287e4eb7d43a7fe/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T07:35:17,035 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 0dafe4c4d07f4913c287e4eb7d43a7fe 2024-11-24T07:35:17,036 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33505/hbase/data/default/testReplayEditsWrittenIntoWAL/0dafe4c4d07f4913c287e4eb7d43a7fe 2024-11-24T07:35:17,037 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33505/hbase/data/default/testReplayEditsWrittenIntoWAL/0dafe4c4d07f4913c287e4eb7d43a7fe 2024-11-24T07:35:17,038 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 0dafe4c4d07f4913c287e4eb7d43a7fe 2024-11-24T07:35:17,038 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 0dafe4c4d07f4913c287e4eb7d43a7fe 2024-11-24T07:35:17,038 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsWrittenIntoWAL descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-24T07:35:17,040 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 0dafe4c4d07f4913c287e4eb7d43a7fe 2024-11-24T07:35:17,042 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:33505/hbase/data/default/testReplayEditsWrittenIntoWAL/0dafe4c4d07f4913c287e4eb7d43a7fe/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-24T07:35:17,043 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 0dafe4c4d07f4913c287e4eb7d43a7fe; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=61371851, jitterRate=-0.08548815548419952}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-24T07:35:17,043 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 0dafe4c4d07f4913c287e4eb7d43a7fe: Writing region info on filesystem at 1732433717027Initializing all the Stores at 1732433717028 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732433717028Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732433717028Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732433717028Cleaning up temporary data from old regions at 1732433717038 (+10 ms)Region opened successfully at 1732433717043 (+5 ms) 2024-11-24T07:35:17,043 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 0dafe4c4d07f4913c287e4eb7d43a7fe, disabling compactions & flushes 2024-11-24T07:35:17,043 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsWrittenIntoWAL,,1732433716992.0dafe4c4d07f4913c287e4eb7d43a7fe. 2024-11-24T07:35:17,043 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsWrittenIntoWAL,,1732433716992.0dafe4c4d07f4913c287e4eb7d43a7fe. 2024-11-24T07:35:17,043 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsWrittenIntoWAL,,1732433716992.0dafe4c4d07f4913c287e4eb7d43a7fe. after waiting 0 ms 2024-11-24T07:35:17,043 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsWrittenIntoWAL,,1732433716992.0dafe4c4d07f4913c287e4eb7d43a7fe. 2024-11-24T07:35:17,044 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsWrittenIntoWAL,,1732433716992.0dafe4c4d07f4913c287e4eb7d43a7fe. 2024-11-24T07:35:17,044 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 0dafe4c4d07f4913c287e4eb7d43a7fe: Waiting for close lock at 1732433717043Disabling compacts and flushes for region at 1732433717043Disabling writes for close at 1732433717043Writing region close event to WAL at 1732433717044 (+1 ms)Closed at 1732433717044 2024-11-24T07:35:17,047 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41971 is added to blk_1073741853_1029 (size=95) 2024-11-24T07:35:17,047 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42685 is added to blk_1073741853_1029 (size=95) 2024-11-24T07:35:17,047 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41161 is added to blk_1073741853_1029 (size=95) 2024-11-24T07:35:17,051 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-11-24T07:35:17,051 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-58985170:(num 1732433716998) 2024-11-24T07:35:17,051 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-24T07:35:17,054 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:33505/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1732433716991, archiveDir=hdfs://localhost:33505/hbase/oldWALs, maxLogs=32 2024-11-24T07:35:17,073 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testreplayeditswrittenintowal-manual,16010,1732433716991/wal.1732433717055, exclude list is [], retry=0 2024-11-24T07:35:17,077 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41971,DS-a67a12dc-5f72-4d11-ac5a-3e225e33c62a,DISK] 2024-11-24T07:35:17,078 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41161,DS-b0260eac-1380-4e84-bf22-a60109fdf5f6,DISK] 2024-11-24T07:35:17,078 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:42685,DS-467ee9a1-f3ca-4279-9b61-69a2a6a0308f,DISK] 2024-11-24T07:35:17,080 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testreplayeditswrittenintowal-manual,16010,1732433716991/wal.1732433717055 2024-11-24T07:35:17,081 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:37111:37111),(127.0.0.1/127.0.0.1:40647:40647),(127.0.0.1/127.0.0.1:35537:35537)] 2024-11-24T07:35:17,316 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:33505/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1732433716991/wal.1732433717055, size=0 (0bytes) 2024-11-24T07:35:17,316 WARN [Time-limited test {}] wal.WALSplitter(453): File hdfs://localhost:33505/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1732433716991/wal.1732433717055 might be still open, length is 0 2024-11-24T07:35:17,316 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:33505/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1732433716991/wal.1732433717055 2024-11-24T07:35:17,317 WARN [IPC Server handler 2 on default port 33505 {}] namenode.FSNamesystem(3854): DIR* NameSystem.internalReleaseLease: File /hbase/WALs/testreplayeditswrittenintowal-manual,16010,1732433716991/wal.1732433717055 has not been closed. Lease recovery is in progress. RecoveryId = 1032 for block blk_1073741855_1031 2024-11-24T07:35:17,318 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=0 on file=hdfs://localhost:33505/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1732433716991/wal.1732433717055 after 2ms 2024-11-24T07:35:17,865 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-166876350_22 at /127.0.0.1:33502 [Receiving block BP-1337235608-172.17.0.3-1732433708251:blk_1073741855_1031] {}] datanode.DataXceiver(331): 127.0.0.1:41971:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:33502 dst: /127.0.0.1:41971 java.io.InterruptedIOException: Interrupted while waiting for IO on channel java.nio.channels.SocketChannel[connected local=localhost/127.0.0.1:41971 remote=/127.0.0.1:33502]. Total timeout mills is 60000, 59402 millis timeout left. at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:350) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?] at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?] at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?] at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?] at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?] at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T07:35:17,866 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-166876350_22 at /127.0.0.1:37408 [Receiving block BP-1337235608-172.17.0.3-1732433708251:blk_1073741855_1031] {}] datanode.DataXceiver(331): 127.0.0.1:42685:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:37408 dst: /127.0.0.1:42685 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T07:35:17,866 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-166876350_22 at /127.0.0.1:55680 [Receiving block BP-1337235608-172.17.0.3-1732433708251:blk_1073741855_1031] {}] datanode.DataXceiver(331): 127.0.0.1:41161:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:55680 dst: /127.0.0.1:41161 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T07:35:17,880 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41971 is added to blk_1073741855_1032 (size=263633) 2024-11-24T07:35:17,882 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41161 is added to blk_1073741855_1032 (size=263633) 2024-11-24T07:35:17,885 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42685 is added to blk_1073741855_1032 (size=263633) 2024-11-24T07:35:21,277 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-24T07:35:21,319 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=1 on file=hdfs://localhost:33505/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1732433716991/wal.1732433717055 after 4003ms 2024-11-24T07:35:21,323 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:33505/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1732433716991/wal.1732433717055: isRecoveredEdits=false, hasTagCompression=true, hasValueCompression=true, valueCompressionType=GZ 2024-11-24T07:35:21,325 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:33505/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1732433716991/wal.1732433717055 took 4008ms 2024-11-24T07:35:21,330 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000001-wal.1732433717055.temp 2024-11-24T07:35:21,332 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta' 2024-11-24T07:35:21,337 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:33505/hbase/data/default/testReplayEditsWrittenIntoWAL/0dafe4c4d07f4913c287e4eb7d43a7fe/recovered.edits/0000000000000000001-wal.1732433717055.temp 2024-11-24T07:35:21,449 INFO [Time-limited test {}] wal.WALSplitter(493): EOF from hdfs://localhost:33505/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1732433716991/wal.1732433717055; continuing. 2024-11-24T07:35:21,449 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:33505/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1732433716991/wal.1732433717055 so closing down 2024-11-24T07:35:21,449 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-24T07:35:21,449 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-11-24T07:35:21,452 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41971 is added to blk_1073741856_1033 (size=263641) 2024-11-24T07:35:21,452 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42685 is added to blk_1073741856_1033 (size=263641) 2024-11-24T07:35:21,453 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41161 is added to blk_1073741856_1033 (size=263641) 2024-11-24T07:35:21,454 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:33505/hbase/data/default/testReplayEditsWrittenIntoWAL/0dafe4c4d07f4913c287e4eb7d43a7fe/recovered.edits/0000000000000000001-wal.1732433717055.temp (wrote 3002 edits, skipped 0 edits in 66 ms) 2024-11-24T07:35:21,456 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:33505/hbase/data/default/testReplayEditsWrittenIntoWAL/0dafe4c4d07f4913c287e4eb7d43a7fe/recovered.edits/0000000000000000001-wal.1732433717055.temp to hdfs://localhost:33505/hbase/data/default/testReplayEditsWrittenIntoWAL/0dafe4c4d07f4913c287e4eb7d43a7fe/recovered.edits/0000000000000003002 2024-11-24T07:35:21,456 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 3002 edits across 1 Regions in 131 ms; skipped=0; WAL=hdfs://localhost:33505/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1732433716991/wal.1732433717055, size=0, length=0, corrupted=false, cancelled=false 2024-11-24T07:35:21,456 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:33505/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1732433716991/wal.1732433717055, journal: Splitting hdfs://localhost:33505/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1732433716991/wal.1732433717055, size=0 (0bytes) at 1732433717316Creating recovered edits writer path=hdfs://localhost:33505/hbase/data/default/testReplayEditsWrittenIntoWAL/0dafe4c4d07f4913c287e4eb7d43a7fe/recovered.edits/0000000000000000001-wal.1732433717055.temp at 1732433721337 (+4021 ms)Split 1024 edits, skipped 0 edits. at 1732433721402 (+65 ms)Split 2048 edits, skipped 0 edits. at 1732433721425 (+23 ms)Finishing writing output for hdfs://localhost:33505/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1732433716991/wal.1732433717055 so closing down at 1732433721449 (+24 ms)3 split writer threads finished at 1732433721449Closed recovered edits writer path=hdfs://localhost:33505/hbase/data/default/testReplayEditsWrittenIntoWAL/0dafe4c4d07f4913c287e4eb7d43a7fe/recovered.edits/0000000000000000001-wal.1732433717055.temp (wrote 3002 edits, skipped 0 edits in 66 ms) at 1732433721454 (+5 ms)Rename recovered edits hdfs://localhost:33505/hbase/data/default/testReplayEditsWrittenIntoWAL/0dafe4c4d07f4913c287e4eb7d43a7fe/recovered.edits/0000000000000000001-wal.1732433717055.temp to hdfs://localhost:33505/hbase/data/default/testReplayEditsWrittenIntoWAL/0dafe4c4d07f4913c287e4eb7d43a7fe/recovered.edits/0000000000000003002 at 1732433721456 (+2 ms)Processed 3002 edits across 1 Regions in 131 ms; skipped=0; WAL=hdfs://localhost:33505/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1732433716991/wal.1732433717055, size=0, length=0, corrupted=false, cancelled=false at 1732433721456 2024-11-24T07:35:21,459 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:33505/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1732433716991/wal.1732433717055 to hdfs://localhost:33505/hbase/oldWALs/wal.1732433717055 2024-11-24T07:35:21,461 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:33505/hbase/data/default/testReplayEditsWrittenIntoWAL/0dafe4c4d07f4913c287e4eb7d43a7fe/recovered.edits/0000000000000003002 2024-11-24T07:35:21,461 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-24T07:35:21,464 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:33505/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1732433716991, archiveDir=hdfs://localhost:33505/hbase/oldWALs, maxLogs=32 2024-11-24T07:35:21,483 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testreplayeditswrittenintowal-manual,16010,1732433716991/wal.1732433721464, exclude list is [], retry=0 2024-11-24T07:35:21,487 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41161,DS-b0260eac-1380-4e84-bf22-a60109fdf5f6,DISK] 2024-11-24T07:35:21,487 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41971,DS-a67a12dc-5f72-4d11-ac5a-3e225e33c62a,DISK] 2024-11-24T07:35:21,488 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:42685,DS-467ee9a1-f3ca-4279-9b61-69a2a6a0308f,DISK] 2024-11-24T07:35:21,490 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testreplayeditswrittenintowal-manual,16010,1732433716991/wal.1732433721464 2024-11-24T07:35:21,490 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:40647:40647),(127.0.0.1/127.0.0.1:37111:37111),(127.0.0.1/127.0.0.1:35537:35537)] 2024-11-24T07:35:21,491 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenIntoWAL,,1732433716992.0dafe4c4d07f4913c287e4eb7d43a7fe.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T07:35:21,493 INFO [StoreOpener-0dafe4c4d07f4913c287e4eb7d43a7fe-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 0dafe4c4d07f4913c287e4eb7d43a7fe 2024-11-24T07:35:21,494 INFO [StoreOpener-0dafe4c4d07f4913c287e4eb7d43a7fe-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 0dafe4c4d07f4913c287e4eb7d43a7fe columnFamilyName a 2024-11-24T07:35:21,494 DEBUG [StoreOpener-0dafe4c4d07f4913c287e4eb7d43a7fe-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T07:35:21,494 INFO [StoreOpener-0dafe4c4d07f4913c287e4eb7d43a7fe-1 {}] regionserver.HStore(327): Store=0dafe4c4d07f4913c287e4eb7d43a7fe/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T07:35:21,495 INFO [StoreOpener-0dafe4c4d07f4913c287e4eb7d43a7fe-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 0dafe4c4d07f4913c287e4eb7d43a7fe 2024-11-24T07:35:21,496 INFO [StoreOpener-0dafe4c4d07f4913c287e4eb7d43a7fe-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 0dafe4c4d07f4913c287e4eb7d43a7fe columnFamilyName b 2024-11-24T07:35:21,496 DEBUG [StoreOpener-0dafe4c4d07f4913c287e4eb7d43a7fe-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T07:35:21,496 INFO [StoreOpener-0dafe4c4d07f4913c287e4eb7d43a7fe-1 {}] regionserver.HStore(327): Store=0dafe4c4d07f4913c287e4eb7d43a7fe/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T07:35:21,497 INFO [StoreOpener-0dafe4c4d07f4913c287e4eb7d43a7fe-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 0dafe4c4d07f4913c287e4eb7d43a7fe 2024-11-24T07:35:21,498 INFO [StoreOpener-0dafe4c4d07f4913c287e4eb7d43a7fe-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 0dafe4c4d07f4913c287e4eb7d43a7fe columnFamilyName c 2024-11-24T07:35:21,498 DEBUG [StoreOpener-0dafe4c4d07f4913c287e4eb7d43a7fe-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T07:35:21,499 INFO [StoreOpener-0dafe4c4d07f4913c287e4eb7d43a7fe-1 {}] regionserver.HStore(327): Store=0dafe4c4d07f4913c287e4eb7d43a7fe/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T07:35:21,499 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 0dafe4c4d07f4913c287e4eb7d43a7fe 2024-11-24T07:35:21,500 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33505/hbase/data/default/testReplayEditsWrittenIntoWAL/0dafe4c4d07f4913c287e4eb7d43a7fe 2024-11-24T07:35:21,503 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:33505/hbase/data/default/testReplayEditsWrittenIntoWAL/0dafe4c4d07f4913c287e4eb7d43a7fe 2024-11-24T07:35:21,504 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:33505/hbase/data/default/testReplayEditsWrittenIntoWAL/0dafe4c4d07f4913c287e4eb7d43a7fe/recovered.edits/0000000000000003002 2024-11-24T07:35:21,507 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:33505/hbase/data/default/testReplayEditsWrittenIntoWAL/0dafe4c4d07f4913c287e4eb7d43a7fe/recovered.edits/0000000000000003002: isRecoveredEdits=true, hasTagCompression=true, hasValueCompression=true, valueCompressionType=GZ 2024-11-24T07:35:21,555 INFO [Time-limited test {}] wal.AbstractTestWALReplay$4$1(796): InternalFlushCache Invoked 2024-11-24T07:35:21,903 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 0dafe4c4d07f4913c287e4eb7d43a7fe 3/3 column families, dataSize=42.49 KB heapSize=100.11 KB 2024-11-24T07:35:21,942 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33505/hbase/data/default/testReplayEditsWrittenIntoWAL/0dafe4c4d07f4913c287e4eb7d43a7fe/.tmp/a/07e766fcafca4ac28c430016d661c969 is 62, key is testReplayEditsWrittenIntoWAL/a:100/1732433717087/Put/seqid=0 2024-11-24T07:35:21,950 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41161 is added to blk_1073741858_1035 (size=50463) 2024-11-24T07:35:21,950 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41971 is added to blk_1073741858_1035 (size=50463) 2024-11-24T07:35:21,950 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42685 is added to blk_1073741858_1035 (size=50463) 2024-11-24T07:35:21,951 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=754 (bloomFilter=true), to=hdfs://localhost:33505/hbase/data/default/testReplayEditsWrittenIntoWAL/0dafe4c4d07f4913c287e4eb7d43a7fe/.tmp/a/07e766fcafca4ac28c430016d661c969 2024-11-24T07:35:21,959 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33505/hbase/data/default/testReplayEditsWrittenIntoWAL/0dafe4c4d07f4913c287e4eb7d43a7fe/.tmp/a/07e766fcafca4ac28c430016d661c969 as hdfs://localhost:33505/hbase/data/default/testReplayEditsWrittenIntoWAL/0dafe4c4d07f4913c287e4eb7d43a7fe/a/07e766fcafca4ac28c430016d661c969 2024-11-24T07:35:21,966 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33505/hbase/data/default/testReplayEditsWrittenIntoWAL/0dafe4c4d07f4913c287e4eb7d43a7fe/a/07e766fcafca4ac28c430016d661c969, entries=754, sequenceid=754, filesize=49.3 K 2024-11-24T07:35:21,966 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~42.49 KB/43512, heapSize ~99.59 KB/101984, currentSize=0 B/0 for 0dafe4c4d07f4913c287e4eb7d43a7fe in 63ms, sequenceid=754, compaction requested=false; wal=null 2024-11-24T07:35:21,997 INFO [Time-limited test {}] wal.AbstractTestWALReplay$4$1(796): InternalFlushCache Invoked 2024-11-24T07:35:21,998 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 0dafe4c4d07f4913c287e4eb7d43a7fe 3/3 column families, dataSize=42.49 KB heapSize=100.11 KB 2024-11-24T07:35:22,008 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33505/hbase/data/default/testReplayEditsWrittenIntoWAL/0dafe4c4d07f4913c287e4eb7d43a7fe/.tmp/a/eb4d7905ccaa4b7d8d52f3db02f63fb0 is 62, key is testReplayEditsWrittenIntoWAL/a:754/1732433717121/Put/seqid=0 2024-11-24T07:35:22,018 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41971 is added to blk_1073741859_1036 (size=20072) 2024-11-24T07:35:22,019 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41161 is added to blk_1073741859_1036 (size=20072) 2024-11-24T07:35:22,019 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42685 is added to blk_1073741859_1036 (size=20072) 2024-11-24T07:35:22,020 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.93 KB at sequenceid=1508 (bloomFilter=true), to=hdfs://localhost:33505/hbase/data/default/testReplayEditsWrittenIntoWAL/0dafe4c4d07f4913c287e4eb7d43a7fe/.tmp/a/eb4d7905ccaa4b7d8d52f3db02f63fb0 2024-11-24T07:35:22,051 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33505/hbase/data/default/testReplayEditsWrittenIntoWAL/0dafe4c4d07f4913c287e4eb7d43a7fe/.tmp/b/0b4dcf27f1e34d5589ff1d2665cdbce8 is 62, key is testReplayEditsWrittenIntoWAL/b:100/1732433717153/Put/seqid=0 2024-11-24T07:35:22,060 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42685 is added to blk_1073741860_1037 (size=35835) 2024-11-24T07:35:22,060 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41971 is added to blk_1073741860_1037 (size=35835) 2024-11-24T07:35:22,060 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41161 is added to blk_1073741860_1037 (size=35835) 2024-11-24T07:35:22,061 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=28.56 KB at sequenceid=1508 (bloomFilter=true), to=hdfs://localhost:33505/hbase/data/default/testReplayEditsWrittenIntoWAL/0dafe4c4d07f4913c287e4eb7d43a7fe/.tmp/b/0b4dcf27f1e34d5589ff1d2665cdbce8 2024-11-24T07:35:22,070 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33505/hbase/data/default/testReplayEditsWrittenIntoWAL/0dafe4c4d07f4913c287e4eb7d43a7fe/.tmp/a/eb4d7905ccaa4b7d8d52f3db02f63fb0 as hdfs://localhost:33505/hbase/data/default/testReplayEditsWrittenIntoWAL/0dafe4c4d07f4913c287e4eb7d43a7fe/a/eb4d7905ccaa4b7d8d52f3db02f63fb0 2024-11-24T07:35:22,079 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33505/hbase/data/default/testReplayEditsWrittenIntoWAL/0dafe4c4d07f4913c287e4eb7d43a7fe/a/eb4d7905ccaa4b7d8d52f3db02f63fb0, entries=246, sequenceid=1508, filesize=19.6 K 2024-11-24T07:35:22,081 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33505/hbase/data/default/testReplayEditsWrittenIntoWAL/0dafe4c4d07f4913c287e4eb7d43a7fe/.tmp/b/0b4dcf27f1e34d5589ff1d2665cdbce8 as hdfs://localhost:33505/hbase/data/default/testReplayEditsWrittenIntoWAL/0dafe4c4d07f4913c287e4eb7d43a7fe/b/0b4dcf27f1e34d5589ff1d2665cdbce8 2024-11-24T07:35:22,089 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33505/hbase/data/default/testReplayEditsWrittenIntoWAL/0dafe4c4d07f4913c287e4eb7d43a7fe/b/0b4dcf27f1e34d5589ff1d2665cdbce8, entries=508, sequenceid=1508, filesize=35.0 K 2024-11-24T07:35:22,090 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~42.49 KB/43512, heapSize ~99.83 KB/102224, currentSize=0 B/0 for 0dafe4c4d07f4913c287e4eb7d43a7fe in 93ms, sequenceid=1508, compaction requested=false; wal=null 2024-11-24T07:35:22,103 INFO [Time-limited test {}] wal.AbstractTestWALReplay$4$1(796): InternalFlushCache Invoked 2024-11-24T07:35:22,104 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 0dafe4c4d07f4913c287e4eb7d43a7fe 3/3 column families, dataSize=42.49 KB heapSize=100.11 KB 2024-11-24T07:35:22,113 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33505/hbase/data/default/testReplayEditsWrittenIntoWAL/0dafe4c4d07f4913c287e4eb7d43a7fe/.tmp/b/e3d5a89086ee495aab912a4c343f87aa is 62, key is testReplayEditsWrittenIntoWAL/b:508/1732433717169/Put/seqid=0 2024-11-24T07:35:22,123 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42685 is added to blk_1073741861_1038 (size=35082) 2024-11-24T07:35:22,123 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41161 is added to blk_1073741861_1038 (size=35082) 2024-11-24T07:35:22,124 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41971 is added to blk_1073741861_1038 (size=35082) 2024-11-24T07:35:22,124 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=27.87 KB at sequenceid=2262 (bloomFilter=true), to=hdfs://localhost:33505/hbase/data/default/testReplayEditsWrittenIntoWAL/0dafe4c4d07f4913c287e4eb7d43a7fe/.tmp/b/e3d5a89086ee495aab912a4c343f87aa 2024-11-24T07:35:22,149 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33505/hbase/data/default/testReplayEditsWrittenIntoWAL/0dafe4c4d07f4913c287e4eb7d43a7fe/.tmp/c/5d417b1a205e4473a41637f2e928a196 is 62, key is testReplayEditsWrittenIntoWAL/c:100/1732433717201/Put/seqid=0 2024-11-24T07:35:22,156 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41971 is added to blk_1073741862_1039 (size=20825) 2024-11-24T07:35:22,157 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42685 is added to blk_1073741862_1039 (size=20825) 2024-11-24T07:35:22,157 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41161 is added to blk_1073741862_1039 (size=20825) 2024-11-24T07:35:22,158 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=14.63 KB at sequenceid=2262 (bloomFilter=true), to=hdfs://localhost:33505/hbase/data/default/testReplayEditsWrittenIntoWAL/0dafe4c4d07f4913c287e4eb7d43a7fe/.tmp/c/5d417b1a205e4473a41637f2e928a196 2024-11-24T07:35:22,165 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33505/hbase/data/default/testReplayEditsWrittenIntoWAL/0dafe4c4d07f4913c287e4eb7d43a7fe/.tmp/b/e3d5a89086ee495aab912a4c343f87aa as hdfs://localhost:33505/hbase/data/default/testReplayEditsWrittenIntoWAL/0dafe4c4d07f4913c287e4eb7d43a7fe/b/e3d5a89086ee495aab912a4c343f87aa 2024-11-24T07:35:22,173 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33505/hbase/data/default/testReplayEditsWrittenIntoWAL/0dafe4c4d07f4913c287e4eb7d43a7fe/b/e3d5a89086ee495aab912a4c343f87aa, entries=492, sequenceid=2262, filesize=34.3 K 2024-11-24T07:35:22,175 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33505/hbase/data/default/testReplayEditsWrittenIntoWAL/0dafe4c4d07f4913c287e4eb7d43a7fe/.tmp/c/5d417b1a205e4473a41637f2e928a196 as hdfs://localhost:33505/hbase/data/default/testReplayEditsWrittenIntoWAL/0dafe4c4d07f4913c287e4eb7d43a7fe/c/5d417b1a205e4473a41637f2e928a196 2024-11-24T07:35:22,182 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33505/hbase/data/default/testReplayEditsWrittenIntoWAL/0dafe4c4d07f4913c287e4eb7d43a7fe/c/5d417b1a205e4473a41637f2e928a196, entries=262, sequenceid=2262, filesize=20.3 K 2024-11-24T07:35:22,182 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~42.49 KB/43512, heapSize ~99.83 KB/102224, currentSize=0 B/0 for 0dafe4c4d07f4913c287e4eb7d43a7fe in 78ms, sequenceid=2262, compaction requested=false; wal=null 2024-11-24T07:35:22,202 WARN [Time-limited test {}] regionserver.HRegion(5722): No family for cell testReplayEditsWrittenIntoWAL/another family:testReplayEditsWrittenIntoWAL/1732433717265/Put/vlen=29/seqid=0 in region testReplayEditsWrittenIntoWAL,,1732433716992.0dafe4c4d07f4913c287e4eb7d43a7fe. 2024-11-24T07:35:22,207 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 3001, skipped 1, firstSequenceIdInLog=1, maxSequenceIdInLog=3002, path=hdfs://localhost:33505/hbase/data/default/testReplayEditsWrittenIntoWAL/0dafe4c4d07f4913c287e4eb7d43a7fe/recovered.edits/0000000000000003002 2024-11-24T07:35:22,207 INFO [Time-limited test {}] wal.AbstractTestWALReplay$4$1(796): InternalFlushCache Invoked 2024-11-24T07:35:22,208 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 0dafe4c4d07f4913c287e4eb7d43a7fe 3/3 column families, dataSize=41.85 KB heapSize=98.89 KB 2024-11-24T07:35:22,218 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33505/hbase/data/default/testReplayEditsWrittenIntoWAL/0dafe4c4d07f4913c287e4eb7d43a7fe/.tmp/c/6e9fcc0565174bc5b7e9ea4ea6743042 is 62, key is testReplayEditsWrittenIntoWAL/c:262/1732433717213/Put/seqid=0 2024-11-24T07:35:22,227 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42685 is added to blk_1073741863_1040 (size=50301) 2024-11-24T07:35:22,228 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41971 is added to blk_1073741863_1040 (size=50301) 2024-11-24T07:35:22,229 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41161 is added to blk_1073741863_1040 (size=50301) 2024-11-24T07:35:22,229 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=41.85 KB at sequenceid=3002 (bloomFilter=true), to=hdfs://localhost:33505/hbase/data/default/testReplayEditsWrittenIntoWAL/0dafe4c4d07f4913c287e4eb7d43a7fe/.tmp/c/6e9fcc0565174bc5b7e9ea4ea6743042 2024-11-24T07:35:22,236 INFO [Time-limited test {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 6e9fcc0565174bc5b7e9ea4ea6743042 2024-11-24T07:35:22,237 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33505/hbase/data/default/testReplayEditsWrittenIntoWAL/0dafe4c4d07f4913c287e4eb7d43a7fe/.tmp/c/6e9fcc0565174bc5b7e9ea4ea6743042 as hdfs://localhost:33505/hbase/data/default/testReplayEditsWrittenIntoWAL/0dafe4c4d07f4913c287e4eb7d43a7fe/c/6e9fcc0565174bc5b7e9ea4ea6743042 2024-11-24T07:35:22,244 INFO [Time-limited test {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 6e9fcc0565174bc5b7e9ea4ea6743042 2024-11-24T07:35:22,244 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33505/hbase/data/default/testReplayEditsWrittenIntoWAL/0dafe4c4d07f4913c287e4eb7d43a7fe/c/6e9fcc0565174bc5b7e9ea4ea6743042, entries=739, sequenceid=3002, filesize=49.1 K 2024-11-24T07:35:22,245 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~41.85 KB/42854, heapSize ~98.38 KB/100736, currentSize=0 B/0 for 0dafe4c4d07f4913c287e4eb7d43a7fe in 38ms, sequenceid=3002, compaction requested=false; wal=null 2024-11-24T07:35:22,246 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:33505/hbase/data/default/testReplayEditsWrittenIntoWAL/0dafe4c4d07f4913c287e4eb7d43a7fe/recovered.edits/0000000000000003002 2024-11-24T07:35:22,247 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 0dafe4c4d07f4913c287e4eb7d43a7fe 2024-11-24T07:35:22,247 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 0dafe4c4d07f4913c287e4eb7d43a7fe 2024-11-24T07:35:22,248 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsWrittenIntoWAL descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-24T07:35:22,250 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 0dafe4c4d07f4913c287e4eb7d43a7fe 2024-11-24T07:35:22,253 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:33505/hbase/data/default/testReplayEditsWrittenIntoWAL/0dafe4c4d07f4913c287e4eb7d43a7fe/recovered.edits/3002.seqid, newMaxSeqId=3002, maxSeqId=1 2024-11-24T07:35:22,254 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 0dafe4c4d07f4913c287e4eb7d43a7fe; next sequenceid=3003; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=204800, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=70249061, jitterRate=0.04679258167743683}}}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-24T07:35:22,254 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 0dafe4c4d07f4913c287e4eb7d43a7fe: Writing region info on filesystem at 1732433721491Initializing all the Stores at 1732433721492 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732433721492Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732433721492Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732433721492Cleaning up temporary data from old regions at 1732433722247 (+755 ms)Region opened successfully at 1732433722254 (+7 ms) 2024-11-24T07:35:22,336 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 0dafe4c4d07f4913c287e4eb7d43a7fe, disabling compactions & flushes 2024-11-24T07:35:22,337 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsWrittenIntoWAL,,1732433716992.0dafe4c4d07f4913c287e4eb7d43a7fe. 2024-11-24T07:35:22,337 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsWrittenIntoWAL,,1732433716992.0dafe4c4d07f4913c287e4eb7d43a7fe. 2024-11-24T07:35:22,337 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsWrittenIntoWAL,,1732433716992.0dafe4c4d07f4913c287e4eb7d43a7fe. after waiting 0 ms 2024-11-24T07:35:22,337 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsWrittenIntoWAL,,1732433716992.0dafe4c4d07f4913c287e4eb7d43a7fe. 2024-11-24T07:35:22,339 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsWrittenIntoWAL,,1732433716992.0dafe4c4d07f4913c287e4eb7d43a7fe. 2024-11-24T07:35:22,339 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 0dafe4c4d07f4913c287e4eb7d43a7fe: Waiting for close lock at 1732433722336Disabling compacts and flushes for region at 1732433722336Disabling writes for close at 1732433722337 (+1 ms)Writing region close event to WAL at 1732433722339 (+2 ms)Closed at 1732433722339 2024-11-24T07:35:22,342 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42685 is added to blk_1073741857_1034 (size=95) 2024-11-24T07:35:22,342 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41971 is added to blk_1073741857_1034 (size=95) 2024-11-24T07:35:22,342 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41161 is added to blk_1073741857_1034 (size=95) 2024-11-24T07:35:22,345 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-11-24T07:35:22,346 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL wal:(num 1732433721464) 2024-11-24T07:35:22,360 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestAsyncWALReplayValueCompression#testReplayEditsWrittenIntoWAL Thread=400 (was 386) Potentially hanging thread: AsyncFSWAL-12-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Client (1319272049) connection to localhost/127.0.0.1:43281 from jenkins java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:43281 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704) java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903) app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: LeaseRenewer:jenkins.replay.wal.secondtime@localhost:33505 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Timer for 'HBase' metrics system java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.util.TimerThread.mainLoop(Timer.java:563) java.base@17.0.11/java.util.TimerThread.run(Timer.java:516) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1536476779_22 at /127.0.0.1:55726 [Waiting for operation #6] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Client (1319272049) connection to localhost/127.0.0.1:33505 from jenkins.replay.wal.secondtime java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) Potentially hanging thread: AsyncFSWAL-12-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1536476779_22 at /127.0.0.1:52694 [Waiting for operation #17] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/089ee269-8889-d2f6-f8a2-4bb8eac365fc/cluster_a7d76481-495d-c9f6-361f-c7016573445a/data/data5 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-12-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Client (1319272049) connection to localhost/127.0.0.1:36885 from jenkins java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1536476779_22 at /127.0.0.1:59906 [Waiting for operation #10] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1536476779_22 at /127.0.0.1:51304 [Waiting for operation #15] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:36885 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704) java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903) app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/089ee269-8889-d2f6-f8a2-4bb8eac365fc/cluster_a7d76481-495d-c9f6-361f-c7016573445a/data/data6 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=865 (was 785) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=247 (was 269), ProcessCount=11 (was 11), AvailableMemoryMB=2308 (was 2369) 2024-11-24T07:35:22,373 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestAsyncWALReplayValueCompression#test2727 Thread=400, OpenFileDescriptor=865, MaxFileDescriptor=1048576, SystemLoadAverage=247, ProcessCount=11, AvailableMemoryMB=2304 2024-11-24T07:35:22,389 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-24T07:35:22,391 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-24T07:35:22,393 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-24T07:35:22,396 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-32460556, suffix=, logDir=hdfs://localhost:33505/hbase/WALs/hregion-32460556, archiveDir=hdfs://localhost:33505/hbase/oldWALs, maxLogs=32 2024-11-24T07:35:22,410 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-32460556/hregion-32460556.1732433722396, exclude list is [], retry=0 2024-11-24T07:35:22,413 DEBUG [AsyncFSWAL-14-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:42685,DS-467ee9a1-f3ca-4279-9b61-69a2a6a0308f,DISK] 2024-11-24T07:35:22,414 DEBUG [AsyncFSWAL-14-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41971,DS-a67a12dc-5f72-4d11-ac5a-3e225e33c62a,DISK] 2024-11-24T07:35:22,414 DEBUG [AsyncFSWAL-14-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41161,DS-b0260eac-1380-4e84-bf22-a60109fdf5f6,DISK] 2024-11-24T07:35:22,417 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-32460556/hregion-32460556.1732433722396 2024-11-24T07:35:22,418 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:35537:35537),(127.0.0.1/127.0.0.1:37111:37111),(127.0.0.1/127.0.0.1:40647:40647)] 2024-11-24T07:35:22,418 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => df7ae2c8214b7226076f3203a9cc6ccb, NAME => 'test2727,,1732433722390.df7ae2c8214b7226076f3203a9cc6ccb.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='test2727', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:33505/hbase 2024-11-24T07:35:22,430 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41161 is added to blk_1073741865_1042 (size=43) 2024-11-24T07:35:22,430 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42685 is added to blk_1073741865_1042 (size=43) 2024-11-24T07:35:22,433 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41971 is added to blk_1073741865_1042 (size=43) 2024-11-24T07:35:22,434 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated test2727,,1732433722390.df7ae2c8214b7226076f3203a9cc6ccb.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T07:35:22,436 INFO [StoreOpener-df7ae2c8214b7226076f3203a9cc6ccb-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region df7ae2c8214b7226076f3203a9cc6ccb 2024-11-24T07:35:22,438 INFO [StoreOpener-df7ae2c8214b7226076f3203a9cc6ccb-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region df7ae2c8214b7226076f3203a9cc6ccb columnFamilyName a 2024-11-24T07:35:22,438 DEBUG [StoreOpener-df7ae2c8214b7226076f3203a9cc6ccb-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T07:35:22,439 INFO [StoreOpener-df7ae2c8214b7226076f3203a9cc6ccb-1 {}] regionserver.HStore(327): Store=df7ae2c8214b7226076f3203a9cc6ccb/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T07:35:22,439 INFO [StoreOpener-df7ae2c8214b7226076f3203a9cc6ccb-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region df7ae2c8214b7226076f3203a9cc6ccb 2024-11-24T07:35:22,441 INFO [StoreOpener-df7ae2c8214b7226076f3203a9cc6ccb-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region df7ae2c8214b7226076f3203a9cc6ccb columnFamilyName b 2024-11-24T07:35:22,441 DEBUG [StoreOpener-df7ae2c8214b7226076f3203a9cc6ccb-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T07:35:22,442 INFO [StoreOpener-df7ae2c8214b7226076f3203a9cc6ccb-1 {}] regionserver.HStore(327): Store=df7ae2c8214b7226076f3203a9cc6ccb/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T07:35:22,442 INFO [StoreOpener-df7ae2c8214b7226076f3203a9cc6ccb-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region df7ae2c8214b7226076f3203a9cc6ccb 2024-11-24T07:35:22,444 INFO [StoreOpener-df7ae2c8214b7226076f3203a9cc6ccb-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region df7ae2c8214b7226076f3203a9cc6ccb columnFamilyName c 2024-11-24T07:35:22,444 DEBUG [StoreOpener-df7ae2c8214b7226076f3203a9cc6ccb-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T07:35:22,444 INFO [StoreOpener-df7ae2c8214b7226076f3203a9cc6ccb-1 {}] regionserver.HStore(327): Store=df7ae2c8214b7226076f3203a9cc6ccb/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T07:35:22,444 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for df7ae2c8214b7226076f3203a9cc6ccb 2024-11-24T07:35:22,445 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33505/hbase/data/default/test2727/df7ae2c8214b7226076f3203a9cc6ccb 2024-11-24T07:35:22,446 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33505/hbase/data/default/test2727/df7ae2c8214b7226076f3203a9cc6ccb 2024-11-24T07:35:22,447 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for df7ae2c8214b7226076f3203a9cc6ccb 2024-11-24T07:35:22,448 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for df7ae2c8214b7226076f3203a9cc6ccb 2024-11-24T07:35:22,448 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table test2727 descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-24T07:35:22,450 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for df7ae2c8214b7226076f3203a9cc6ccb 2024-11-24T07:35:22,452 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:33505/hbase/data/default/test2727/df7ae2c8214b7226076f3203a9cc6ccb/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-24T07:35:22,453 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened df7ae2c8214b7226076f3203a9cc6ccb; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=73287614, jitterRate=0.0920705497264862}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-24T07:35:22,454 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for df7ae2c8214b7226076f3203a9cc6ccb: Writing region info on filesystem at 1732433722434Initializing all the Stores at 1732433722435 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732433722435Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732433722436 (+1 ms)Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732433722436Cleaning up temporary data from old regions at 1732433722448 (+12 ms)Region opened successfully at 1732433722453 (+5 ms) 2024-11-24T07:35:22,454 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing df7ae2c8214b7226076f3203a9cc6ccb, disabling compactions & flushes 2024-11-24T07:35:22,454 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region test2727,,1732433722390.df7ae2c8214b7226076f3203a9cc6ccb. 2024-11-24T07:35:22,454 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on test2727,,1732433722390.df7ae2c8214b7226076f3203a9cc6ccb. 2024-11-24T07:35:22,454 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on test2727,,1732433722390.df7ae2c8214b7226076f3203a9cc6ccb. after waiting 0 ms 2024-11-24T07:35:22,454 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region test2727,,1732433722390.df7ae2c8214b7226076f3203a9cc6ccb. 2024-11-24T07:35:22,455 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed test2727,,1732433722390.df7ae2c8214b7226076f3203a9cc6ccb. 2024-11-24T07:35:22,455 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for df7ae2c8214b7226076f3203a9cc6ccb: Waiting for close lock at 1732433722454Disabling compacts and flushes for region at 1732433722454Disabling writes for close at 1732433722454Writing region close event to WAL at 1732433722455 (+1 ms)Closed at 1732433722455 2024-11-24T07:35:22,458 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41971 is added to blk_1073741864_1041 (size=95) 2024-11-24T07:35:22,458 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42685 is added to blk_1073741864_1041 (size=95) 2024-11-24T07:35:22,459 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41161 is added to blk_1073741864_1041 (size=95) 2024-11-24T07:35:22,461 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-11-24T07:35:22,461 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-32460556:(num 1732433722396) 2024-11-24T07:35:22,461 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-24T07:35:22,464 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:33505/hbase/WALs/test2727-manual,16010,1732433722388, archiveDir=hdfs://localhost:33505/hbase/oldWALs, maxLogs=32 2024-11-24T07:35:22,477 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/test2727-manual,16010,1732433722388/wal.1732433722465, exclude list is [], retry=0 2024-11-24T07:35:22,481 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:42685,DS-467ee9a1-f3ca-4279-9b61-69a2a6a0308f,DISK] 2024-11-24T07:35:22,481 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41971,DS-a67a12dc-5f72-4d11-ac5a-3e225e33c62a,DISK] 2024-11-24T07:35:22,482 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41161,DS-b0260eac-1380-4e84-bf22-a60109fdf5f6,DISK] 2024-11-24T07:35:22,483 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/test2727-manual,16010,1732433722388/wal.1732433722465 2024-11-24T07:35:22,484 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:35537:35537),(127.0.0.1/127.0.0.1:37111:37111),(127.0.0.1/127.0.0.1:40647:40647)] 2024-11-24T07:35:22,624 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41971 is added to blk_1073741866_1043 (size=263359) 2024-11-24T07:35:22,625 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41161 is added to blk_1073741866_1043 (size=263359) 2024-11-24T07:35:22,625 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42685 is added to blk_1073741866_1043 (size=263359) 2024-11-24T07:35:22,643 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:33505/hbase/WALs/test2727-manual,16010,1732433722388/wal.1732433722465, size=257.2 K (263359bytes) 2024-11-24T07:35:22,643 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:33505/hbase/WALs/test2727-manual,16010,1732433722388/wal.1732433722465 2024-11-24T07:35:22,644 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:33505/hbase/WALs/test2727-manual,16010,1732433722388/wal.1732433722465 after 1ms 2024-11-24T07:35:22,647 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:33505/hbase/WALs/test2727-manual,16010,1732433722388/wal.1732433722465: isRecoveredEdits=false, hasTagCompression=true, hasValueCompression=true, valueCompressionType=GZ 2024-11-24T07:35:22,648 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:33505/hbase/WALs/test2727-manual,16010,1732433722388/wal.1732433722465 took 5ms 2024-11-24T07:35:22,654 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000001-wal.1732433722465.temp 2024-11-24T07:35:22,656 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:33505/hbase/data/default/test2727/df7ae2c8214b7226076f3203a9cc6ccb/recovered.edits/0000000000000000001-wal.1732433722465.temp 2024-11-24T07:35:22,736 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:33505/hbase/WALs/test2727-manual,16010,1732433722388/wal.1732433722465 so closing down 2024-11-24T07:35:22,736 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-24T07:35:22,737 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-11-24T07:35:22,740 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41971 is added to blk_1073741867_1044 (size=263359) 2024-11-24T07:35:22,741 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42685 is added to blk_1073741867_1044 (size=263359) 2024-11-24T07:35:22,741 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41161 is added to blk_1073741867_1044 (size=263359) 2024-11-24T07:35:22,742 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:33505/hbase/data/default/test2727/df7ae2c8214b7226076f3203a9cc6ccb/recovered.edits/0000000000000000001-wal.1732433722465.temp (wrote 3000 edits, skipped 0 edits in 72 ms) 2024-11-24T07:35:22,744 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:33505/hbase/data/default/test2727/df7ae2c8214b7226076f3203a9cc6ccb/recovered.edits/0000000000000000001-wal.1732433722465.temp to hdfs://localhost:33505/hbase/data/default/test2727/df7ae2c8214b7226076f3203a9cc6ccb/recovered.edits/0000000000000003000 2024-11-24T07:35:22,744 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 3000 edits across 1 Regions in 95 ms; skipped=0; WAL=hdfs://localhost:33505/hbase/WALs/test2727-manual,16010,1732433722388/wal.1732433722465, size=257.2 K, length=263359, corrupted=false, cancelled=false 2024-11-24T07:35:22,745 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:33505/hbase/WALs/test2727-manual,16010,1732433722388/wal.1732433722465, journal: Splitting hdfs://localhost:33505/hbase/WALs/test2727-manual,16010,1732433722388/wal.1732433722465, size=257.2 K (263359bytes) at 1732433722643Creating recovered edits writer path=hdfs://localhost:33505/hbase/data/default/test2727/df7ae2c8214b7226076f3203a9cc6ccb/recovered.edits/0000000000000000001-wal.1732433722465.temp at 1732433722656 (+13 ms)Split 1024 edits, skipped 0 edits. at 1732433722671 (+15 ms)Split 2048 edits, skipped 0 edits. at 1732433722687 (+16 ms)Finishing writing output for hdfs://localhost:33505/hbase/WALs/test2727-manual,16010,1732433722388/wal.1732433722465 so closing down at 1732433722736 (+49 ms)3 split writer threads finished at 1732433722737 (+1 ms)Closed recovered edits writer path=hdfs://localhost:33505/hbase/data/default/test2727/df7ae2c8214b7226076f3203a9cc6ccb/recovered.edits/0000000000000000001-wal.1732433722465.temp (wrote 3000 edits, skipped 0 edits in 72 ms) at 1732433722742 (+5 ms)Rename recovered edits hdfs://localhost:33505/hbase/data/default/test2727/df7ae2c8214b7226076f3203a9cc6ccb/recovered.edits/0000000000000000001-wal.1732433722465.temp to hdfs://localhost:33505/hbase/data/default/test2727/df7ae2c8214b7226076f3203a9cc6ccb/recovered.edits/0000000000000003000 at 1732433722744 (+2 ms)Processed 3000 edits across 1 Regions in 95 ms; skipped=0; WAL=hdfs://localhost:33505/hbase/WALs/test2727-manual,16010,1732433722388/wal.1732433722465, size=257.2 K, length=263359, corrupted=false, cancelled=false at 1732433722744 2024-11-24T07:35:22,747 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:33505/hbase/WALs/test2727-manual,16010,1732433722388/wal.1732433722465 to hdfs://localhost:33505/hbase/oldWALs/wal.1732433722465 2024-11-24T07:35:22,748 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:33505/hbase/data/default/test2727/df7ae2c8214b7226076f3203a9cc6ccb/recovered.edits/0000000000000003000 2024-11-24T07:35:22,748 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-24T07:35:22,751 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:33505/hbase/WALs/test2727-manual,16010,1732433722388, archiveDir=hdfs://localhost:33505/hbase/oldWALs, maxLogs=32 2024-11-24T07:35:22,774 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/test2727-manual,16010,1732433722388/wal.1732433722751, exclude list is [], retry=0 2024-11-24T07:35:22,779 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:42685,DS-467ee9a1-f3ca-4279-9b61-69a2a6a0308f,DISK] 2024-11-24T07:35:22,779 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41971,DS-a67a12dc-5f72-4d11-ac5a-3e225e33c62a,DISK] 2024-11-24T07:35:22,780 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41161,DS-b0260eac-1380-4e84-bf22-a60109fdf5f6,DISK] 2024-11-24T07:35:22,787 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/test2727-manual,16010,1732433722388/wal.1732433722751 2024-11-24T07:35:22,788 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:35537:35537),(127.0.0.1/127.0.0.1:37111:37111),(127.0.0.1/127.0.0.1:40647:40647)] 2024-11-24T07:35:22,979 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42685 is added to blk_1073741868_1045 (size=263486) 2024-11-24T07:35:22,980 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41971 is added to blk_1073741868_1045 (size=263486) 2024-11-24T07:35:22,980 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41161 is added to blk_1073741868_1045 (size=263486) 2024-11-24T07:35:22,999 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:33505/hbase/WALs/test2727-manual,16010,1732433722388/wal.1732433722751, size=257.3 K (263486bytes) 2024-11-24T07:35:22,999 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:33505/hbase/WALs/test2727-manual,16010,1732433722388/wal.1732433722751 2024-11-24T07:35:23,000 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:33505/hbase/WALs/test2727-manual,16010,1732433722388/wal.1732433722751 after 0ms 2024-11-24T07:35:23,003 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:33505/hbase/WALs/test2727-manual,16010,1732433722388/wal.1732433722751: isRecoveredEdits=false, hasTagCompression=true, hasValueCompression=true, valueCompressionType=GZ 2024-11-24T07:35:23,005 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:33505/hbase/WALs/test2727-manual,16010,1732433722388/wal.1732433722751 took 6ms 2024-11-24T07:35:23,008 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000003001-wal.1732433722751.temp 2024-11-24T07:35:23,010 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:33505/hbase/data/default/test2727/df7ae2c8214b7226076f3203a9cc6ccb/recovered.edits/0000000000000003001-wal.1732433722751.temp 2024-11-24T07:35:23,051 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta 2024-11-24T07:35:23,052 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_meta Metrics about Tables on a single HBase RegionServer 2024-11-24T07:35:23,054 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-24T07:35:23,054 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint Metrics about HBase RegionObservers 2024-11-24T07:35:23,054 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:33505/hbase/WALs/test2727-manual,16010,1732433722388/wal.1732433722751 so closing down 2024-11-24T07:35:23,054 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-24T07:35:23,054 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_master_table_store 2024-11-24T07:35:23,055 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_master_table_store Metrics about Tables on a single HBase RegionServer 2024-11-24T07:35:23,055 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-11-24T07:35:23,056 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testReplayEditsWrittenIntoWAL 2024-11-24T07:35:23,056 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testReplayEditsWrittenIntoWAL Metrics about Tables on a single HBase RegionServer 2024-11-24T07:35:23,058 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41161 is added to blk_1073741869_1046 (size=263486) 2024-11-24T07:35:23,059 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42685 is added to blk_1073741869_1046 (size=263486) 2024-11-24T07:35:23,059 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41971 is added to blk_1073741869_1046 (size=263486) 2024-11-24T07:35:23,060 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:33505/hbase/data/default/test2727/df7ae2c8214b7226076f3203a9cc6ccb/recovered.edits/0000000000000003001-wal.1732433722751.temp (wrote 3000 edits, skipped 0 edits in 33 ms) 2024-11-24T07:35:23,061 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:33505/hbase/data/default/test2727/df7ae2c8214b7226076f3203a9cc6ccb/recovered.edits/0000000000000003001-wal.1732433722751.temp to hdfs://localhost:33505/hbase/data/default/test2727/df7ae2c8214b7226076f3203a9cc6ccb/recovered.edits/0000000000000006000 2024-11-24T07:35:23,062 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 3000 edits across 1 Regions in 57 ms; skipped=0; WAL=hdfs://localhost:33505/hbase/WALs/test2727-manual,16010,1732433722388/wal.1732433722751, size=257.3 K, length=263486, corrupted=false, cancelled=false 2024-11-24T07:35:23,062 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:33505/hbase/WALs/test2727-manual,16010,1732433722388/wal.1732433722751, journal: Splitting hdfs://localhost:33505/hbase/WALs/test2727-manual,16010,1732433722388/wal.1732433722751, size=257.3 K (263486bytes) at 1732433722999Creating recovered edits writer path=hdfs://localhost:33505/hbase/data/default/test2727/df7ae2c8214b7226076f3203a9cc6ccb/recovered.edits/0000000000000003001-wal.1732433722751.temp at 1732433723010 (+11 ms)Split 1024 edits, skipped 0 edits. at 1732433723024 (+14 ms)Split 2048 edits, skipped 0 edits. at 1732433723040 (+16 ms)Finishing writing output for hdfs://localhost:33505/hbase/WALs/test2727-manual,16010,1732433722388/wal.1732433722751 so closing down at 1732433723054 (+14 ms)3 split writer threads finished at 1732433723055 (+1 ms)Closed recovered edits writer path=hdfs://localhost:33505/hbase/data/default/test2727/df7ae2c8214b7226076f3203a9cc6ccb/recovered.edits/0000000000000003001-wal.1732433722751.temp (wrote 3000 edits, skipped 0 edits in 33 ms) at 1732433723060 (+5 ms)Rename recovered edits hdfs://localhost:33505/hbase/data/default/test2727/df7ae2c8214b7226076f3203a9cc6ccb/recovered.edits/0000000000000003001-wal.1732433722751.temp to hdfs://localhost:33505/hbase/data/default/test2727/df7ae2c8214b7226076f3203a9cc6ccb/recovered.edits/0000000000000006000 at 1732433723061 (+1 ms)Processed 3000 edits across 1 Regions in 57 ms; skipped=0; WAL=hdfs://localhost:33505/hbase/WALs/test2727-manual,16010,1732433722388/wal.1732433722751, size=257.3 K, length=263486, corrupted=false, cancelled=false at 1732433723062 (+1 ms) 2024-11-24T07:35:23,064 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:33505/hbase/WALs/test2727-manual,16010,1732433722388/wal.1732433722751 to hdfs://localhost:33505/hbase/oldWALs/wal.1732433722751 2024-11-24T07:35:23,065 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:33505/hbase/data/default/test2727/df7ae2c8214b7226076f3203a9cc6ccb/recovered.edits/0000000000000006000 2024-11-24T07:35:23,065 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-24T07:35:23,067 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:33505/hbase/WALs/test2727-manual,16010,1732433722388, archiveDir=hdfs://localhost:33505/hbase/oldWALs, maxLogs=32 2024-11-24T07:35:23,080 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/test2727-manual,16010,1732433722388/wal.1732433723067, exclude list is [], retry=0 2024-11-24T07:35:23,083 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41971,DS-a67a12dc-5f72-4d11-ac5a-3e225e33c62a,DISK] 2024-11-24T07:35:23,084 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:42685,DS-467ee9a1-f3ca-4279-9b61-69a2a6a0308f,DISK] 2024-11-24T07:35:23,084 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41161,DS-b0260eac-1380-4e84-bf22-a60109fdf5f6,DISK] 2024-11-24T07:35:23,087 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/test2727-manual,16010,1732433722388/wal.1732433723067 2024-11-24T07:35:23,087 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:37111:37111),(127.0.0.1/127.0.0.1:35537:35537),(127.0.0.1/127.0.0.1:40647:40647)] 2024-11-24T07:35:23,087 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => df7ae2c8214b7226076f3203a9cc6ccb, NAME => 'test2727,,1732433722390.df7ae2c8214b7226076f3203a9cc6ccb.', STARTKEY => '', ENDKEY => ''} 2024-11-24T07:35:23,087 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated test2727,,1732433722390.df7ae2c8214b7226076f3203a9cc6ccb.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T07:35:23,088 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for df7ae2c8214b7226076f3203a9cc6ccb 2024-11-24T07:35:23,088 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for df7ae2c8214b7226076f3203a9cc6ccb 2024-11-24T07:35:23,090 INFO [StoreOpener-df7ae2c8214b7226076f3203a9cc6ccb-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region df7ae2c8214b7226076f3203a9cc6ccb 2024-11-24T07:35:23,091 INFO [StoreOpener-df7ae2c8214b7226076f3203a9cc6ccb-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region df7ae2c8214b7226076f3203a9cc6ccb columnFamilyName a 2024-11-24T07:35:23,091 DEBUG [StoreOpener-df7ae2c8214b7226076f3203a9cc6ccb-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T07:35:23,092 INFO [StoreOpener-df7ae2c8214b7226076f3203a9cc6ccb-1 {}] regionserver.HStore(327): Store=df7ae2c8214b7226076f3203a9cc6ccb/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T07:35:23,092 INFO [StoreOpener-df7ae2c8214b7226076f3203a9cc6ccb-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region df7ae2c8214b7226076f3203a9cc6ccb 2024-11-24T07:35:23,094 INFO [StoreOpener-df7ae2c8214b7226076f3203a9cc6ccb-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region df7ae2c8214b7226076f3203a9cc6ccb columnFamilyName b 2024-11-24T07:35:23,094 DEBUG [StoreOpener-df7ae2c8214b7226076f3203a9cc6ccb-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T07:35:23,094 INFO [StoreOpener-df7ae2c8214b7226076f3203a9cc6ccb-1 {}] regionserver.HStore(327): Store=df7ae2c8214b7226076f3203a9cc6ccb/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T07:35:23,095 INFO [StoreOpener-df7ae2c8214b7226076f3203a9cc6ccb-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region df7ae2c8214b7226076f3203a9cc6ccb 2024-11-24T07:35:23,096 INFO [StoreOpener-df7ae2c8214b7226076f3203a9cc6ccb-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region df7ae2c8214b7226076f3203a9cc6ccb columnFamilyName c 2024-11-24T07:35:23,096 DEBUG [StoreOpener-df7ae2c8214b7226076f3203a9cc6ccb-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T07:35:23,097 INFO [StoreOpener-df7ae2c8214b7226076f3203a9cc6ccb-1 {}] regionserver.HStore(327): Store=df7ae2c8214b7226076f3203a9cc6ccb/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T07:35:23,097 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for df7ae2c8214b7226076f3203a9cc6ccb 2024-11-24T07:35:23,098 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33505/hbase/data/default/test2727/df7ae2c8214b7226076f3203a9cc6ccb 2024-11-24T07:35:23,100 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 2 recovered edits file(s) under hdfs://localhost:33505/hbase/data/default/test2727/df7ae2c8214b7226076f3203a9cc6ccb 2024-11-24T07:35:23,101 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:33505/hbase/data/default/test2727/df7ae2c8214b7226076f3203a9cc6ccb/recovered.edits/0000000000000003000 2024-11-24T07:35:23,103 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:33505/hbase/data/default/test2727/df7ae2c8214b7226076f3203a9cc6ccb/recovered.edits/0000000000000003000: isRecoveredEdits=true, hasTagCompression=true, hasValueCompression=true, valueCompressionType=GZ 2024-11-24T07:35:23,140 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 3000, skipped 0, firstSequenceIdInLog=1, maxSequenceIdInLog=3000, path=hdfs://localhost:33505/hbase/data/default/test2727/df7ae2c8214b7226076f3203a9cc6ccb/recovered.edits/0000000000000003000 2024-11-24T07:35:23,141 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:33505/hbase/data/default/test2727/df7ae2c8214b7226076f3203a9cc6ccb/recovered.edits/0000000000000006000 2024-11-24T07:35:23,143 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:33505/hbase/data/default/test2727/df7ae2c8214b7226076f3203a9cc6ccb/recovered.edits/0000000000000006000: isRecoveredEdits=true, hasTagCompression=true, hasValueCompression=true, valueCompressionType=GZ 2024-11-24T07:35:23,180 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 3000, skipped 0, firstSequenceIdInLog=3001, maxSequenceIdInLog=6000, path=hdfs://localhost:33505/hbase/data/default/test2727/df7ae2c8214b7226076f3203a9cc6ccb/recovered.edits/0000000000000006000 2024-11-24T07:35:23,180 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing df7ae2c8214b7226076f3203a9cc6ccb 3/3 column families, dataSize=215.51 KB heapSize=657 KB 2024-11-24T07:35:23,201 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33505/hbase/data/default/test2727/df7ae2c8214b7226076f3203a9cc6ccb/.tmp/a/fa6dadaa4de942d383c51d8bb7605703 is 41, key is test2727/a:100/1732433722794/Put/seqid=0 2024-11-24T07:35:23,209 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41161 is added to blk_1073741871_1048 (size=84227) 2024-11-24T07:35:23,210 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41971 is added to blk_1073741871_1048 (size=84227) 2024-11-24T07:35:23,210 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42685 is added to blk_1073741871_1048 (size=84227) 2024-11-24T07:35:23,210 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=71.84 KB at sequenceid=6000 (bloomFilter=true), to=hdfs://localhost:33505/hbase/data/default/test2727/df7ae2c8214b7226076f3203a9cc6ccb/.tmp/a/fa6dadaa4de942d383c51d8bb7605703 2024-11-24T07:35:23,236 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33505/hbase/data/default/test2727/df7ae2c8214b7226076f3203a9cc6ccb/.tmp/b/8553760efe604f5baaabc40d3e6200a6 is 41, key is test2727/b:100/1732433722859/Put/seqid=0 2024-11-24T07:35:23,242 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41971 is added to blk_1073741872_1049 (size=84609) 2024-11-24T07:35:23,242 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41161 is added to blk_1073741872_1049 (size=84609) 2024-11-24T07:35:23,243 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42685 is added to blk_1073741872_1049 (size=84609) 2024-11-24T07:35:23,244 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=71.84 KB at sequenceid=6000 (bloomFilter=true), to=hdfs://localhost:33505/hbase/data/default/test2727/df7ae2c8214b7226076f3203a9cc6ccb/.tmp/b/8553760efe604f5baaabc40d3e6200a6 2024-11-24T07:35:23,280 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33505/hbase/data/default/test2727/df7ae2c8214b7226076f3203a9cc6ccb/.tmp/c/467368560dd34f00b7e8c8f1b766eb27 is 41, key is test2727/c:100/1732433722920/Put/seqid=0 2024-11-24T07:35:23,285 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41161 is added to blk_1073741873_1050 (size=84609) 2024-11-24T07:35:23,286 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41971 is added to blk_1073741873_1050 (size=84609) 2024-11-24T07:35:23,286 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42685 is added to blk_1073741873_1050 (size=84609) 2024-11-24T07:35:23,288 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=71.84 KB at sequenceid=6000 (bloomFilter=true), to=hdfs://localhost:33505/hbase/data/default/test2727/df7ae2c8214b7226076f3203a9cc6ccb/.tmp/c/467368560dd34f00b7e8c8f1b766eb27 2024-11-24T07:35:23,295 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33505/hbase/data/default/test2727/df7ae2c8214b7226076f3203a9cc6ccb/.tmp/a/fa6dadaa4de942d383c51d8bb7605703 as hdfs://localhost:33505/hbase/data/default/test2727/df7ae2c8214b7226076f3203a9cc6ccb/a/fa6dadaa4de942d383c51d8bb7605703 2024-11-24T07:35:23,302 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33505/hbase/data/default/test2727/df7ae2c8214b7226076f3203a9cc6ccb/a/fa6dadaa4de942d383c51d8bb7605703, entries=2000, sequenceid=6000, filesize=82.3 K 2024-11-24T07:35:23,304 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33505/hbase/data/default/test2727/df7ae2c8214b7226076f3203a9cc6ccb/.tmp/b/8553760efe604f5baaabc40d3e6200a6 as hdfs://localhost:33505/hbase/data/default/test2727/df7ae2c8214b7226076f3203a9cc6ccb/b/8553760efe604f5baaabc40d3e6200a6 2024-11-24T07:35:23,311 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33505/hbase/data/default/test2727/df7ae2c8214b7226076f3203a9cc6ccb/b/8553760efe604f5baaabc40d3e6200a6, entries=2000, sequenceid=6000, filesize=82.6 K 2024-11-24T07:35:23,313 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33505/hbase/data/default/test2727/df7ae2c8214b7226076f3203a9cc6ccb/.tmp/c/467368560dd34f00b7e8c8f1b766eb27 as hdfs://localhost:33505/hbase/data/default/test2727/df7ae2c8214b7226076f3203a9cc6ccb/c/467368560dd34f00b7e8c8f1b766eb27 2024-11-24T07:35:23,320 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33505/hbase/data/default/test2727/df7ae2c8214b7226076f3203a9cc6ccb/c/467368560dd34f00b7e8c8f1b766eb27, entries=2000, sequenceid=6000, filesize=82.6 K 2024-11-24T07:35:23,320 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~215.51 KB/220680, heapSize ~656.95 KB/672720, currentSize=0 B/0 for df7ae2c8214b7226076f3203a9cc6ccb in 140ms, sequenceid=6000, compaction requested=false; wal=null 2024-11-24T07:35:23,321 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:33505/hbase/data/default/test2727/df7ae2c8214b7226076f3203a9cc6ccb/recovered.edits/0000000000000003000 2024-11-24T07:35:23,322 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:33505/hbase/data/default/test2727/df7ae2c8214b7226076f3203a9cc6ccb/recovered.edits/0000000000000006000 2024-11-24T07:35:23,323 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for df7ae2c8214b7226076f3203a9cc6ccb 2024-11-24T07:35:23,323 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for df7ae2c8214b7226076f3203a9cc6ccb 2024-11-24T07:35:23,324 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table test2727 descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-24T07:35:23,325 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for df7ae2c8214b7226076f3203a9cc6ccb 2024-11-24T07:35:23,328 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:33505/hbase/data/default/test2727/df7ae2c8214b7226076f3203a9cc6ccb/recovered.edits/6000.seqid, newMaxSeqId=6000, maxSeqId=1 2024-11-24T07:35:23,329 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened df7ae2c8214b7226076f3203a9cc6ccb; next sequenceid=6001; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=69081429, jitterRate=0.029393509030342102}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-24T07:35:23,330 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for df7ae2c8214b7226076f3203a9cc6ccb: Writing region info on filesystem at 1732433723088Initializing all the Stores at 1732433723089 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732433723089Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732433723089Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732433723089Obtaining lock to block concurrent updates at 1732433723180 (+91 ms)Preparing flush snapshotting stores in df7ae2c8214b7226076f3203a9cc6ccb at 1732433723180Finished memstore snapshotting test2727,,1732433722390.df7ae2c8214b7226076f3203a9cc6ccb., syncing WAL and waiting on mvcc, flushsize=dataSize=220680, getHeapSize=672720, getOffHeapSize=0, getCellsCount=6000 at 1732433723181 (+1 ms)Flushing stores of test2727,,1732433722390.df7ae2c8214b7226076f3203a9cc6ccb. at 1732433723181Flushing df7ae2c8214b7226076f3203a9cc6ccb/a: creating writer at 1732433723181Flushing df7ae2c8214b7226076f3203a9cc6ccb/a: appending metadata at 1732433723200 (+19 ms)Flushing df7ae2c8214b7226076f3203a9cc6ccb/a: closing flushed file at 1732433723200Flushing df7ae2c8214b7226076f3203a9cc6ccb/b: creating writer at 1732433723217 (+17 ms)Flushing df7ae2c8214b7226076f3203a9cc6ccb/b: appending metadata at 1732433723235 (+18 ms)Flushing df7ae2c8214b7226076f3203a9cc6ccb/b: closing flushed file at 1732433723235Flushing df7ae2c8214b7226076f3203a9cc6ccb/c: creating writer at 1732433723252 (+17 ms)Flushing df7ae2c8214b7226076f3203a9cc6ccb/c: appending metadata at 1732433723279 (+27 ms)Flushing df7ae2c8214b7226076f3203a9cc6ccb/c: closing flushed file at 1732433723279Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@572ed6c3: reopening flushed file at 1732433723294 (+15 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@6e79b729: reopening flushed file at 1732433723303 (+9 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@2ba81efe: reopening flushed file at 1732433723311 (+8 ms)Finished flush of dataSize ~215.51 KB/220680, heapSize ~656.95 KB/672720, currentSize=0 B/0 for df7ae2c8214b7226076f3203a9cc6ccb in 140ms, sequenceid=6000, compaction requested=false; wal=null at 1732433723320 (+9 ms)Cleaning up temporary data from old regions at 1732433723323 (+3 ms)Region opened successfully at 1732433723329 (+6 ms) 2024-11-24T07:35:23,330 DEBUG [Time-limited test {}] wal.AbstractTestWALReplay(320): region.getOpenSeqNum(): 6001, wal3.id: 0 2024-11-24T07:35:23,331 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing df7ae2c8214b7226076f3203a9cc6ccb, disabling compactions & flushes 2024-11-24T07:35:23,331 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region test2727,,1732433722390.df7ae2c8214b7226076f3203a9cc6ccb. 2024-11-24T07:35:23,331 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on test2727,,1732433722390.df7ae2c8214b7226076f3203a9cc6ccb. 2024-11-24T07:35:23,331 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on test2727,,1732433722390.df7ae2c8214b7226076f3203a9cc6ccb. after waiting 0 ms 2024-11-24T07:35:23,331 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region test2727,,1732433722390.df7ae2c8214b7226076f3203a9cc6ccb. 2024-11-24T07:35:23,332 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed test2727,,1732433722390.df7ae2c8214b7226076f3203a9cc6ccb. 2024-11-24T07:35:23,332 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for df7ae2c8214b7226076f3203a9cc6ccb: Waiting for close lock at 1732433723331Disabling compacts and flushes for region at 1732433723331Disabling writes for close at 1732433723331Writing region close event to WAL at 1732433723332 (+1 ms)Closed at 1732433723332 2024-11-24T07:35:23,336 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41971 is added to blk_1073741870_1047 (size=95) 2024-11-24T07:35:23,336 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41161 is added to blk_1073741870_1047 (size=95) 2024-11-24T07:35:23,336 WARN [Close-WAL-Writer-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(650): complete file /hbase/WALs/test2727-manual,16010,1732433722388/wal.1732433723067 not finished, retry = 0 2024-11-24T07:35:23,336 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42685 is added to blk_1073741870_1047 (size=95) 2024-11-24T07:35:23,439 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-11-24T07:35:23,439 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL wal:(num 1732433723067) 2024-11-24T07:35:23,451 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestAsyncWALReplayValueCompression#test2727 Thread=403 (was 400) Potentially hanging thread: AsyncFSWAL-14-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-14-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-166876350_22 at /127.0.0.1:51434 [Waiting for operation #13] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-166876350_22 at /127.0.0.1:52828 [Waiting for operation #5] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-14-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=929 (was 865) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=247 (was 247), ProcessCount=11 (was 11), AvailableMemoryMB=2182 (was 2304) 2024-11-24T07:35:23,462 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestAsyncWALReplayValueCompression#testSequentialEditLogSeqNum Thread=403, OpenFileDescriptor=929, MaxFileDescriptor=1048576, SystemLoadAverage=247, ProcessCount=11, AvailableMemoryMB=2181 2024-11-24T07:35:23,476 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-24T07:35:23,483 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:33505/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1732433723476, archiveDir=hdfs://localhost:33505/hbase/oldWALs, maxLogs=32 2024-11-24T07:35:23,483 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732433723483 2024-11-24T07:35:23,490 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testsequentialeditlogseqnum-manual,16010,1732433723476/wal.1732433723483 2024-11-24T07:35:23,493 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new MockWAL writer with pipeline: [(127.0.0.1/127.0.0.1:37111:37111),(127.0.0.1/127.0.0.1:35537:35537),(127.0.0.1/127.0.0.1:40647:40647)] 2024-11-24T07:35:23,497 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 8e134a0c00c3bdd8c85f167e63c65d3e, NAME => 'testSequentialEditLogSeqNum,,1732433723477.8e134a0c00c3bdd8c85f167e63c65d3e.', STARTKEY => '', ENDKEY => ''} 2024-11-24T07:35:23,498 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testSequentialEditLogSeqNum,,1732433723477.8e134a0c00c3bdd8c85f167e63c65d3e.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T07:35:23,498 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 8e134a0c00c3bdd8c85f167e63c65d3e 2024-11-24T07:35:23,498 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 8e134a0c00c3bdd8c85f167e63c65d3e 2024-11-24T07:35:23,499 WARN [Time-limited test {}] regionserver.HRegionFileSystem(836): hdfs://localhost:33505/hbase/data/default/testSequentialEditLogSeqNum/8e134a0c00c3bdd8c85f167e63c65d3e doesn't exist for region: 8e134a0c00c3bdd8c85f167e63c65d3e on table testSequentialEditLogSeqNum 2024-11-24T07:35:23,500 WARN [Time-limited test {}] regionserver.HRegionFileSystem(854): .regioninfo file not found for region: 8e134a0c00c3bdd8c85f167e63c65d3e on table testSequentialEditLogSeqNum 2024-11-24T07:35:23,510 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42685 is added to blk_1073741875_1052 (size=62) 2024-11-24T07:35:23,511 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41161 is added to blk_1073741875_1052 (size=62) 2024-11-24T07:35:23,511 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41971 is added to blk_1073741875_1052 (size=62) 2024-11-24T07:35:23,513 INFO [StoreOpener-8e134a0c00c3bdd8c85f167e63c65d3e-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 8e134a0c00c3bdd8c85f167e63c65d3e 2024-11-24T07:35:23,515 INFO [StoreOpener-8e134a0c00c3bdd8c85f167e63c65d3e-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 8e134a0c00c3bdd8c85f167e63c65d3e columnFamilyName a 2024-11-24T07:35:23,515 DEBUG [StoreOpener-8e134a0c00c3bdd8c85f167e63c65d3e-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T07:35:23,515 INFO [StoreOpener-8e134a0c00c3bdd8c85f167e63c65d3e-1 {}] regionserver.HStore(327): Store=8e134a0c00c3bdd8c85f167e63c65d3e/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T07:35:23,515 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 8e134a0c00c3bdd8c85f167e63c65d3e 2024-11-24T07:35:23,516 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33505/hbase/data/default/testSequentialEditLogSeqNum/8e134a0c00c3bdd8c85f167e63c65d3e 2024-11-24T07:35:23,516 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33505/hbase/data/default/testSequentialEditLogSeqNum/8e134a0c00c3bdd8c85f167e63c65d3e 2024-11-24T07:35:23,517 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 8e134a0c00c3bdd8c85f167e63c65d3e 2024-11-24T07:35:23,517 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 8e134a0c00c3bdd8c85f167e63c65d3e 2024-11-24T07:35:23,519 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 8e134a0c00c3bdd8c85f167e63c65d3e 2024-11-24T07:35:23,521 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:33505/hbase/data/default/testSequentialEditLogSeqNum/8e134a0c00c3bdd8c85f167e63c65d3e/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-24T07:35:23,521 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 8e134a0c00c3bdd8c85f167e63c65d3e; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=72479807, jitterRate=0.08003328740596771}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-24T07:35:23,522 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 8e134a0c00c3bdd8c85f167e63c65d3e: Writing region info on filesystem at 1732433723498Initializing all the Stores at 1732433723512 (+14 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732433723512Cleaning up temporary data from old regions at 1732433723517 (+5 ms)Region opened successfully at 1732433723522 (+5 ms) 2024-11-24T07:35:23,533 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 8e134a0c00c3bdd8c85f167e63c65d3e 1/1 column families, dataSize=770 B heapSize=1.73 KB 2024-11-24T07:35:23,554 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33505/hbase/data/default/testSequentialEditLogSeqNum/8e134a0c00c3bdd8c85f167e63c65d3e/.tmp/a/9c0a36250e324a8686e3d1f6bc5cd6ea is 81, key is testSequentialEditLogSeqNum/a:x0/1732433723522/Put/seqid=0 2024-11-24T07:35:23,560 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41971 is added to blk_1073741876_1053 (size=5833) 2024-11-24T07:35:23,561 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41161 is added to blk_1073741876_1053 (size=5833) 2024-11-24T07:35:23,562 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=770 B at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:33505/hbase/data/default/testSequentialEditLogSeqNum/8e134a0c00c3bdd8c85f167e63c65d3e/.tmp/a/9c0a36250e324a8686e3d1f6bc5cd6ea 2024-11-24T07:35:23,563 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42685 is added to blk_1073741876_1053 (size=5833) 2024-11-24T07:35:23,569 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33505/hbase/data/default/testSequentialEditLogSeqNum/8e134a0c00c3bdd8c85f167e63c65d3e/.tmp/a/9c0a36250e324a8686e3d1f6bc5cd6ea as hdfs://localhost:33505/hbase/data/default/testSequentialEditLogSeqNum/8e134a0c00c3bdd8c85f167e63c65d3e/a/9c0a36250e324a8686e3d1f6bc5cd6ea 2024-11-24T07:35:23,576 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33505/hbase/data/default/testSequentialEditLogSeqNum/8e134a0c00c3bdd8c85f167e63c65d3e/a/9c0a36250e324a8686e3d1f6bc5cd6ea, entries=10, sequenceid=13, filesize=5.7 K 2024-11-24T07:35:23,577 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~770 B/770, heapSize ~1.72 KB/1760, currentSize=0 B/0 for 8e134a0c00c3bdd8c85f167e63c65d3e in 44ms, sequenceid=13, compaction requested=false 2024-11-24T07:35:23,577 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for 8e134a0c00c3bdd8c85f167e63c65d3e: 2024-11-24T07:35:23,583 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T07:35:23,583 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T07:35:23,583 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T07:35:23,583 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T07:35:23,583 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T07:35:23,586 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41161 is added to blk_1073741874_1051 (size=1844) 2024-11-24T07:35:23,586 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42685 is added to blk_1073741874_1051 (size=1844) 2024-11-24T07:35:23,587 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41971 is added to blk_1073741874_1051 (size=1844) 2024-11-24T07:35:23,603 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:33505/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1732433723476/wal.1732433723483, size=1.8 K (1844bytes) 2024-11-24T07:35:23,603 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:33505/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1732433723476/wal.1732433723483 2024-11-24T07:35:23,604 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:33505/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1732433723476/wal.1732433723483 after 1ms 2024-11-24T07:35:23,607 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:33505/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1732433723476/wal.1732433723483: isRecoveredEdits=false, hasTagCompression=true, hasValueCompression=true, valueCompressionType=GZ 2024-11-24T07:35:23,607 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:33505/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1732433723476/wal.1732433723483 took 4ms 2024-11-24T07:35:23,610 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:33505/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1732433723476/wal.1732433723483 so closing down 2024-11-24T07:35:23,610 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-24T07:35:23,611 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000003-wal.1732433723483.temp 2024-11-24T07:35:23,612 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:33505/hbase/data/default/testSequentialEditLogSeqNum/8e134a0c00c3bdd8c85f167e63c65d3e/recovered.edits/0000000000000000003-wal.1732433723483.temp 2024-11-24T07:35:23,613 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-11-24T07:35:23,622 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41971 is added to blk_1073741877_1054 (size=1477) 2024-11-24T07:35:23,622 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41161 is added to blk_1073741877_1054 (size=1477) 2024-11-24T07:35:23,622 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42685 is added to blk_1073741877_1054 (size=1477) 2024-11-24T07:35:23,623 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:33505/hbase/data/default/testSequentialEditLogSeqNum/8e134a0c00c3bdd8c85f167e63c65d3e/recovered.edits/0000000000000000003-wal.1732433723483.temp (wrote 15 edits, skipped 0 edits in 0 ms) 2024-11-24T07:35:23,625 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:33505/hbase/data/default/testSequentialEditLogSeqNum/8e134a0c00c3bdd8c85f167e63c65d3e/recovered.edits/0000000000000000003-wal.1732433723483.temp to hdfs://localhost:33505/hbase/data/default/testSequentialEditLogSeqNum/8e134a0c00c3bdd8c85f167e63c65d3e/recovered.edits/0000000000000000020 2024-11-24T07:35:23,625 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 17 edits across 1 Regions in 18 ms; skipped=2; WAL=hdfs://localhost:33505/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1732433723476/wal.1732433723483, size=1.8 K, length=1844, corrupted=false, cancelled=false 2024-11-24T07:35:23,625 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:33505/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1732433723476/wal.1732433723483, journal: Splitting hdfs://localhost:33505/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1732433723476/wal.1732433723483, size=1.8 K (1844bytes) at 1732433723603Finishing writing output for hdfs://localhost:33505/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1732433723476/wal.1732433723483 so closing down at 1732433723610 (+7 ms)Creating recovered edits writer path=hdfs://localhost:33505/hbase/data/default/testSequentialEditLogSeqNum/8e134a0c00c3bdd8c85f167e63c65d3e/recovered.edits/0000000000000000003-wal.1732433723483.temp at 1732433723612 (+2 ms)3 split writer threads finished at 1732433723613 (+1 ms)Closed recovered edits writer path=hdfs://localhost:33505/hbase/data/default/testSequentialEditLogSeqNum/8e134a0c00c3bdd8c85f167e63c65d3e/recovered.edits/0000000000000000003-wal.1732433723483.temp (wrote 15 edits, skipped 0 edits in 0 ms) at 1732433723623 (+10 ms)Rename recovered edits hdfs://localhost:33505/hbase/data/default/testSequentialEditLogSeqNum/8e134a0c00c3bdd8c85f167e63c65d3e/recovered.edits/0000000000000000003-wal.1732433723483.temp to hdfs://localhost:33505/hbase/data/default/testSequentialEditLogSeqNum/8e134a0c00c3bdd8c85f167e63c65d3e/recovered.edits/0000000000000000020 at 1732433723625 (+2 ms)Processed 17 edits across 1 Regions in 18 ms; skipped=2; WAL=hdfs://localhost:33505/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1732433723476/wal.1732433723483, size=1.8 K, length=1844, corrupted=false, cancelled=false at 1732433723625 2024-11-24T07:35:23,638 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestAsyncWALReplayValueCompression#testSequentialEditLogSeqNum Thread=408 (was 403) Potentially hanging thread: sync.4 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-166876350_22 at /127.0.0.1:51434 [Waiting for operation #15] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-166876350_22 at /127.0.0.1:55726 [Waiting for operation #7] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: sync.0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-166876350_22 at /127.0.0.1:52828 [Waiting for operation #7] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: sync.1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441) Potentially hanging thread: sync.3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-166876350_22 at /127.0.0.1:59906 [Waiting for operation #11] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: sync.2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441) - Thread LEAK? -, OpenFileDescriptor=967 (was 929) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=247 (was 247), ProcessCount=11 (was 11), AvailableMemoryMB=2177 (was 2181) 2024-11-24T07:35:23,649 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestAsyncWALReplayValueCompression#testRegionMadeOfBulkLoadedFilesOnly Thread=408, OpenFileDescriptor=967, MaxFileDescriptor=1048576, SystemLoadAverage=247, ProcessCount=11, AvailableMemoryMB=2177 2024-11-24T07:35:23,664 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-24T07:35:23,666 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-24T07:35:23,693 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-24T07:35:23,696 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-99980262, suffix=, logDir=hdfs://localhost:33505/hbase/WALs/hregion-99980262, archiveDir=hdfs://localhost:33505/hbase/oldWALs, maxLogs=32 2024-11-24T07:35:23,709 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-99980262/hregion-99980262.1732433723696, exclude list is [], retry=0 2024-11-24T07:35:23,712 DEBUG [AsyncFSWAL-17-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:42685,DS-467ee9a1-f3ca-4279-9b61-69a2a6a0308f,DISK] 2024-11-24T07:35:23,712 DEBUG [AsyncFSWAL-17-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41161,DS-b0260eac-1380-4e84-bf22-a60109fdf5f6,DISK] 2024-11-24T07:35:23,712 DEBUG [AsyncFSWAL-17-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41971,DS-a67a12dc-5f72-4d11-ac5a-3e225e33c62a,DISK] 2024-11-24T07:35:23,715 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-99980262/hregion-99980262.1732433723696 2024-11-24T07:35:23,715 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:35537:35537),(127.0.0.1/127.0.0.1:40647:40647),(127.0.0.1/127.0.0.1:37111:37111)] 2024-11-24T07:35:23,716 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => b25495aaf2c830ef51e5e99e180edf06, NAME => 'testRegionMadeOfBulkLoadedFilesOnly,,1732433723665.b25495aaf2c830ef51e5e99e180edf06.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testRegionMadeOfBulkLoadedFilesOnly', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:33505/hbase 2024-11-24T07:35:23,728 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41971 is added to blk_1073741879_1056 (size=70) 2024-11-24T07:35:23,729 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42685 is added to blk_1073741879_1056 (size=70) 2024-11-24T07:35:23,729 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41161 is added to blk_1073741879_1056 (size=70) 2024-11-24T07:35:23,730 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testRegionMadeOfBulkLoadedFilesOnly,,1732433723665.b25495aaf2c830ef51e5e99e180edf06.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T07:35:23,731 INFO [StoreOpener-b25495aaf2c830ef51e5e99e180edf06-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region b25495aaf2c830ef51e5e99e180edf06 2024-11-24T07:35:23,733 INFO [StoreOpener-b25495aaf2c830ef51e5e99e180edf06-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region b25495aaf2c830ef51e5e99e180edf06 columnFamilyName a 2024-11-24T07:35:23,733 DEBUG [StoreOpener-b25495aaf2c830ef51e5e99e180edf06-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T07:35:23,734 INFO [StoreOpener-b25495aaf2c830ef51e5e99e180edf06-1 {}] regionserver.HStore(327): Store=b25495aaf2c830ef51e5e99e180edf06/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T07:35:23,734 INFO [StoreOpener-b25495aaf2c830ef51e5e99e180edf06-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region b25495aaf2c830ef51e5e99e180edf06 2024-11-24T07:35:23,736 INFO [StoreOpener-b25495aaf2c830ef51e5e99e180edf06-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region b25495aaf2c830ef51e5e99e180edf06 columnFamilyName b 2024-11-24T07:35:23,736 DEBUG [StoreOpener-b25495aaf2c830ef51e5e99e180edf06-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T07:35:23,736 INFO [StoreOpener-b25495aaf2c830ef51e5e99e180edf06-1 {}] regionserver.HStore(327): Store=b25495aaf2c830ef51e5e99e180edf06/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T07:35:23,737 INFO [StoreOpener-b25495aaf2c830ef51e5e99e180edf06-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region b25495aaf2c830ef51e5e99e180edf06 2024-11-24T07:35:23,738 INFO [StoreOpener-b25495aaf2c830ef51e5e99e180edf06-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region b25495aaf2c830ef51e5e99e180edf06 columnFamilyName c 2024-11-24T07:35:23,739 DEBUG [StoreOpener-b25495aaf2c830ef51e5e99e180edf06-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T07:35:23,739 INFO [StoreOpener-b25495aaf2c830ef51e5e99e180edf06-1 {}] regionserver.HStore(327): Store=b25495aaf2c830ef51e5e99e180edf06/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T07:35:23,739 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for b25495aaf2c830ef51e5e99e180edf06 2024-11-24T07:35:23,740 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33505/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/b25495aaf2c830ef51e5e99e180edf06 2024-11-24T07:35:23,741 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33505/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/b25495aaf2c830ef51e5e99e180edf06 2024-11-24T07:35:23,742 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for b25495aaf2c830ef51e5e99e180edf06 2024-11-24T07:35:23,742 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for b25495aaf2c830ef51e5e99e180edf06 2024-11-24T07:35:23,743 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testRegionMadeOfBulkLoadedFilesOnly descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-24T07:35:23,744 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for b25495aaf2c830ef51e5e99e180edf06 2024-11-24T07:35:23,747 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:33505/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/b25495aaf2c830ef51e5e99e180edf06/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-24T07:35:23,748 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened b25495aaf2c830ef51e5e99e180edf06; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=68409215, jitterRate=0.019376739859580994}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-24T07:35:23,749 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for b25495aaf2c830ef51e5e99e180edf06: Writing region info on filesystem at 1732433723730Initializing all the Stores at 1732433723731 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732433723731Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732433723731Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732433723731Cleaning up temporary data from old regions at 1732433723742 (+11 ms)Region opened successfully at 1732433723749 (+7 ms) 2024-11-24T07:35:23,749 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing b25495aaf2c830ef51e5e99e180edf06, disabling compactions & flushes 2024-11-24T07:35:23,749 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testRegionMadeOfBulkLoadedFilesOnly,,1732433723665.b25495aaf2c830ef51e5e99e180edf06. 2024-11-24T07:35:23,749 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testRegionMadeOfBulkLoadedFilesOnly,,1732433723665.b25495aaf2c830ef51e5e99e180edf06. 2024-11-24T07:35:23,749 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testRegionMadeOfBulkLoadedFilesOnly,,1732433723665.b25495aaf2c830ef51e5e99e180edf06. after waiting 0 ms 2024-11-24T07:35:23,749 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testRegionMadeOfBulkLoadedFilesOnly,,1732433723665.b25495aaf2c830ef51e5e99e180edf06. 2024-11-24T07:35:23,750 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testRegionMadeOfBulkLoadedFilesOnly,,1732433723665.b25495aaf2c830ef51e5e99e180edf06. 2024-11-24T07:35:23,750 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for b25495aaf2c830ef51e5e99e180edf06: Waiting for close lock at 1732433723749Disabling compacts and flushes for region at 1732433723749Disabling writes for close at 1732433723749Writing region close event to WAL at 1732433723750 (+1 ms)Closed at 1732433723750 2024-11-24T07:35:23,753 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41971 is added to blk_1073741878_1055 (size=95) 2024-11-24T07:35:23,753 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41161 is added to blk_1073741878_1055 (size=95) 2024-11-24T07:35:23,753 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42685 is added to blk_1073741878_1055 (size=95) 2024-11-24T07:35:23,756 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-11-24T07:35:23,756 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-99980262:(num 1732433723696) 2024-11-24T07:35:23,756 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-24T07:35:23,758 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:33505/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1732433723663, archiveDir=hdfs://localhost:33505/hbase/oldWALs, maxLogs=32 2024-11-24T07:35:23,772 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1732433723663/wal.1732433723759, exclude list is [], retry=0 2024-11-24T07:35:23,775 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41971,DS-a67a12dc-5f72-4d11-ac5a-3e225e33c62a,DISK] 2024-11-24T07:35:23,776 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41161,DS-b0260eac-1380-4e84-bf22-a60109fdf5f6,DISK] 2024-11-24T07:35:23,776 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:42685,DS-467ee9a1-f3ca-4279-9b61-69a2a6a0308f,DISK] 2024-11-24T07:35:23,778 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1732433723663/wal.1732433723759 2024-11-24T07:35:23,779 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:37111:37111),(127.0.0.1/127.0.0.1:40647:40647),(127.0.0.1/127.0.0.1:35537:35537)] 2024-11-24T07:35:23,779 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => b25495aaf2c830ef51e5e99e180edf06, NAME => 'testRegionMadeOfBulkLoadedFilesOnly,,1732433723665.b25495aaf2c830ef51e5e99e180edf06.', STARTKEY => '', ENDKEY => ''} 2024-11-24T07:35:23,779 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testRegionMadeOfBulkLoadedFilesOnly,,1732433723665.b25495aaf2c830ef51e5e99e180edf06.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T07:35:23,780 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for b25495aaf2c830ef51e5e99e180edf06 2024-11-24T07:35:23,780 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for b25495aaf2c830ef51e5e99e180edf06 2024-11-24T07:35:23,782 INFO [StoreOpener-b25495aaf2c830ef51e5e99e180edf06-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region b25495aaf2c830ef51e5e99e180edf06 2024-11-24T07:35:23,783 INFO [StoreOpener-b25495aaf2c830ef51e5e99e180edf06-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region b25495aaf2c830ef51e5e99e180edf06 columnFamilyName a 2024-11-24T07:35:23,783 DEBUG [StoreOpener-b25495aaf2c830ef51e5e99e180edf06-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T07:35:23,784 INFO [StoreOpener-b25495aaf2c830ef51e5e99e180edf06-1 {}] regionserver.HStore(327): Store=b25495aaf2c830ef51e5e99e180edf06/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T07:35:23,784 INFO [StoreOpener-b25495aaf2c830ef51e5e99e180edf06-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region b25495aaf2c830ef51e5e99e180edf06 2024-11-24T07:35:23,785 INFO [StoreOpener-b25495aaf2c830ef51e5e99e180edf06-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region b25495aaf2c830ef51e5e99e180edf06 columnFamilyName b 2024-11-24T07:35:23,785 DEBUG [StoreOpener-b25495aaf2c830ef51e5e99e180edf06-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T07:35:23,786 INFO [StoreOpener-b25495aaf2c830ef51e5e99e180edf06-1 {}] regionserver.HStore(327): Store=b25495aaf2c830ef51e5e99e180edf06/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T07:35:23,786 INFO [StoreOpener-b25495aaf2c830ef51e5e99e180edf06-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region b25495aaf2c830ef51e5e99e180edf06 2024-11-24T07:35:23,787 INFO [StoreOpener-b25495aaf2c830ef51e5e99e180edf06-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region b25495aaf2c830ef51e5e99e180edf06 columnFamilyName c 2024-11-24T07:35:23,787 DEBUG [StoreOpener-b25495aaf2c830ef51e5e99e180edf06-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T07:35:23,788 INFO [StoreOpener-b25495aaf2c830ef51e5e99e180edf06-1 {}] regionserver.HStore(327): Store=b25495aaf2c830ef51e5e99e180edf06/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T07:35:23,788 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for b25495aaf2c830ef51e5e99e180edf06 2024-11-24T07:35:23,789 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33505/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/b25495aaf2c830ef51e5e99e180edf06 2024-11-24T07:35:23,790 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33505/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/b25495aaf2c830ef51e5e99e180edf06 2024-11-24T07:35:23,791 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for b25495aaf2c830ef51e5e99e180edf06 2024-11-24T07:35:23,791 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for b25495aaf2c830ef51e5e99e180edf06 2024-11-24T07:35:23,792 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testRegionMadeOfBulkLoadedFilesOnly descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-24T07:35:23,794 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for b25495aaf2c830ef51e5e99e180edf06 2024-11-24T07:35:23,794 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened b25495aaf2c830ef51e5e99e180edf06; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=72365913, jitterRate=0.07833613455295563}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-24T07:35:23,795 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for b25495aaf2c830ef51e5e99e180edf06: Writing region info on filesystem at 1732433723780Initializing all the Stores at 1732433723781 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732433723781Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732433723781Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732433723781Cleaning up temporary data from old regions at 1732433723791 (+10 ms)Region opened successfully at 1732433723795 (+4 ms) 2024-11-24T07:35:23,800 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33505/hbase/testRegionMadeOfBulkLoadedFilesOnly/hfile is 28, key is \x0D/a:a/1732433723798/Put/seqid=0 2024-11-24T07:35:23,808 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42685 is added to blk_1073741881_1058 (size=4826) 2024-11-24T07:35:23,808 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41971 is added to blk_1073741881_1058 (size=4826) 2024-11-24T07:35:23,808 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41161 is added to blk_1073741881_1058 (size=4826) 2024-11-24T07:35:23,809 INFO [Time-limited test {}] regionserver.HStore(614): Validating hfile at hdfs://localhost:33505/hbase/testRegionMadeOfBulkLoadedFilesOnly/hfile for inclusion in b25495aaf2c830ef51e5e99e180edf06/a 2024-11-24T07:35:23,818 DEBUG [Time-limited test {}] regionserver.HStore(626): HFile bounds: first= last=z 2024-11-24T07:35:23,818 DEBUG [Time-limited test {}] regionserver.HStore(628): Region bounds: first= last= 2024-11-24T07:35:23,818 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for b25495aaf2c830ef51e5e99e180edf06: 2024-11-24T07:35:23,820 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33505/hbase/testRegionMadeOfBulkLoadedFilesOnly/hfile as hdfs://localhost:33505/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/b25495aaf2c830ef51e5e99e180edf06/a/dfa272b15dc7460195b1762575a439c1_SeqId_3_ 2024-11-24T07:35:23,821 INFO [Time-limited test {}] regionserver.HStore(700): Loaded HFile hdfs://localhost:33505/hbase/testRegionMadeOfBulkLoadedFilesOnly/hfile into b25495aaf2c830ef51e5e99e180edf06/a as hdfs://localhost:33505/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/b25495aaf2c830ef51e5e99e180edf06/a/dfa272b15dc7460195b1762575a439c1_SeqId_3_ - updating store file list. 2024-11-24T07:35:23,829 DEBUG [Time-limited test {}] regionserver.HStoreFile(483): HFile Bloom filter type for dfa272b15dc7460195b1762575a439c1_SeqId_3_: NONE, but ROW specified in column family configuration 2024-11-24T07:35:23,829 INFO [Time-limited test {}] regionserver.HStore(722): Loaded HFile hdfs://localhost:33505/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/b25495aaf2c830ef51e5e99e180edf06/a/dfa272b15dc7460195b1762575a439c1_SeqId_3_ into b25495aaf2c830ef51e5e99e180edf06/a 2024-11-24T07:35:23,830 INFO [Time-limited test {}] regionserver.HStore(706): Successfully loaded hdfs://localhost:33505/hbase/testRegionMadeOfBulkLoadedFilesOnly/hfile into b25495aaf2c830ef51e5e99e180edf06/a (new location: hdfs://localhost:33505/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/b25495aaf2c830ef51e5e99e180edf06/a/dfa272b15dc7460195b1762575a439c1_SeqId_3_) 2024-11-24T07:35:23,877 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:33505/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1732433723663/wal.1732433723759, size=0 (0bytes) 2024-11-24T07:35:23,877 WARN [Time-limited test {}] wal.WALSplitter(453): File hdfs://localhost:33505/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1732433723663/wal.1732433723759 might be still open, length is 0 2024-11-24T07:35:23,877 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:33505/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1732433723663/wal.1732433723759 2024-11-24T07:35:23,878 WARN [IPC Server handler 0 on default port 33505 {}] namenode.FSNamesystem(3854): DIR* NameSystem.internalReleaseLease: File /hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1732433723663/wal.1732433723759 has not been closed. Lease recovery is in progress. RecoveryId = 1059 for block blk_1073741880_1057 2024-11-24T07:35:23,878 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=0 on file=hdfs://localhost:33505/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1732433723663/wal.1732433723759 after 1ms 2024-11-24T07:35:26,853 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-166876350_22 at /127.0.0.1:52914 [Receiving block BP-1337235608-172.17.0.3-1732433708251:blk_1073741880_1057] {}] datanode.DataXceiver(331): 127.0.0.1:41971:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:52914 dst: /127.0.0.1:41971 java.io.InterruptedIOException: Interrupted while waiting for IO on channel java.nio.channels.SocketChannel[connected local=localhost/127.0.0.1:41971 remote=/127.0.0.1:52914]. Total timeout mills is 60000, 56985 millis timeout left. at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:350) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?] at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?] at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?] at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?] at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?] at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T07:35:26,853 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-166876350_22 at /127.0.0.1:51518 [Receiving block BP-1337235608-172.17.0.3-1732433708251:blk_1073741880_1057] {}] datanode.DataXceiver(331): 127.0.0.1:42685:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:51518 dst: /127.0.0.1:42685 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T07:35:26,853 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-166876350_22 at /127.0.0.1:60098 [Receiving block BP-1337235608-172.17.0.3-1732433708251:blk_1073741880_1057] {}] datanode.DataXceiver(331): 127.0.0.1:41161:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:60098 dst: /127.0.0.1:41161 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T07:35:26,859 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41971 is added to blk_1073741880_1059 (size=474) 2024-11-24T07:35:26,860 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41161 is added to blk_1073741880_1059 (size=474) 2024-11-24T07:35:27,879 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=1 on file=hdfs://localhost:33505/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1732433723663/wal.1732433723759 after 4002ms 2024-11-24T07:35:27,888 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:33505/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1732433723663/wal.1732433723759: isRecoveredEdits=false, hasTagCompression=true, hasValueCompression=true, valueCompressionType=GZ 2024-11-24T07:35:27,888 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:33505/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1732433723663/wal.1732433723759 took 4011ms 2024-11-24T07:35:27,891 INFO [Time-limited test {}] wal.WALSplitter(493): EOF from hdfs://localhost:33505/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1732433723663/wal.1732433723759; continuing. 2024-11-24T07:35:27,891 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:33505/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1732433723663/wal.1732433723759 so closing down 2024-11-24T07:35:27,891 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-24T07:35:27,893 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000005-wal.1732433723759.temp 2024-11-24T07:35:27,895 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:33505/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/b25495aaf2c830ef51e5e99e180edf06/recovered.edits/0000000000000000005-wal.1732433723759.temp 2024-11-24T07:35:27,895 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-11-24T07:35:27,902 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42685 is added to blk_1073741882_1060 (size=259) 2024-11-24T07:35:27,903 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41161 is added to blk_1073741882_1060 (size=259) 2024-11-24T07:35:27,903 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41971 is added to blk_1073741882_1060 (size=259) 2024-11-24T07:35:27,903 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:33505/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/b25495aaf2c830ef51e5e99e180edf06/recovered.edits/0000000000000000005-wal.1732433723759.temp (wrote 1 edits, skipped 0 edits in 0 ms) 2024-11-24T07:35:27,905 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:33505/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/b25495aaf2c830ef51e5e99e180edf06/recovered.edits/0000000000000000005-wal.1732433723759.temp to hdfs://localhost:33505/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/b25495aaf2c830ef51e5e99e180edf06/recovered.edits/0000000000000000005 2024-11-24T07:35:27,905 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 2 edits across 1 Regions in 16 ms; skipped=1; WAL=hdfs://localhost:33505/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1732433723663/wal.1732433723759, size=0, length=0, corrupted=false, cancelled=false 2024-11-24T07:35:27,905 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:33505/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1732433723663/wal.1732433723759, journal: Splitting hdfs://localhost:33505/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1732433723663/wal.1732433723759, size=0 (0bytes) at 1732433723877Finishing writing output for hdfs://localhost:33505/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1732433723663/wal.1732433723759 so closing down at 1732433727891 (+4014 ms)Creating recovered edits writer path=hdfs://localhost:33505/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/b25495aaf2c830ef51e5e99e180edf06/recovered.edits/0000000000000000005-wal.1732433723759.temp at 1732433727895 (+4 ms)3 split writer threads finished at 1732433727895Closed recovered edits writer path=hdfs://localhost:33505/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/b25495aaf2c830ef51e5e99e180edf06/recovered.edits/0000000000000000005-wal.1732433723759.temp (wrote 1 edits, skipped 0 edits in 0 ms) at 1732433727903 (+8 ms)Rename recovered edits hdfs://localhost:33505/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/b25495aaf2c830ef51e5e99e180edf06/recovered.edits/0000000000000000005-wal.1732433723759.temp to hdfs://localhost:33505/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/b25495aaf2c830ef51e5e99e180edf06/recovered.edits/0000000000000000005 at 1732433727905 (+2 ms)Processed 2 edits across 1 Regions in 16 ms; skipped=1; WAL=hdfs://localhost:33505/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1732433723663/wal.1732433723759, size=0, length=0, corrupted=false, cancelled=false at 1732433727905 2024-11-24T07:35:27,907 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:33505/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1732433723663/wal.1732433723759 to hdfs://localhost:33505/hbase/oldWALs/wal.1732433723759 2024-11-24T07:35:27,908 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:33505/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/b25495aaf2c830ef51e5e99e180edf06/recovered.edits/0000000000000000005 2024-11-24T07:35:27,908 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-24T07:35:27,910 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:33505/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1732433723663, archiveDir=hdfs://localhost:33505/hbase/oldWALs, maxLogs=32 2024-11-24T07:35:27,924 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1732433723663/wal.1732433727910, exclude list is [], retry=0 2024-11-24T07:35:27,927 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:42685,DS-467ee9a1-f3ca-4279-9b61-69a2a6a0308f,DISK] 2024-11-24T07:35:27,927 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41161,DS-b0260eac-1380-4e84-bf22-a60109fdf5f6,DISK] 2024-11-24T07:35:27,928 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41971,DS-a67a12dc-5f72-4d11-ac5a-3e225e33c62a,DISK] 2024-11-24T07:35:27,929 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1732433723663/wal.1732433727910 2024-11-24T07:35:27,930 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:35537:35537),(127.0.0.1/127.0.0.1:40647:40647),(127.0.0.1/127.0.0.1:37111:37111)] 2024-11-24T07:35:27,930 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => b25495aaf2c830ef51e5e99e180edf06, NAME => 'testRegionMadeOfBulkLoadedFilesOnly,,1732433723665.b25495aaf2c830ef51e5e99e180edf06.', STARTKEY => '', ENDKEY => ''} 2024-11-24T07:35:27,930 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testRegionMadeOfBulkLoadedFilesOnly,,1732433723665.b25495aaf2c830ef51e5e99e180edf06.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T07:35:27,930 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for b25495aaf2c830ef51e5e99e180edf06 2024-11-24T07:35:27,930 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for b25495aaf2c830ef51e5e99e180edf06 2024-11-24T07:35:27,932 INFO [StoreOpener-b25495aaf2c830ef51e5e99e180edf06-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region b25495aaf2c830ef51e5e99e180edf06 2024-11-24T07:35:27,934 INFO [StoreOpener-b25495aaf2c830ef51e5e99e180edf06-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region b25495aaf2c830ef51e5e99e180edf06 columnFamilyName a 2024-11-24T07:35:27,934 DEBUG [StoreOpener-b25495aaf2c830ef51e5e99e180edf06-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T07:35:27,941 DEBUG [StoreFileOpener-b25495aaf2c830ef51e5e99e180edf06-a-1 {}] regionserver.HStoreFile(483): HFile Bloom filter type for dfa272b15dc7460195b1762575a439c1_SeqId_3_: NONE, but ROW specified in column family configuration 2024-11-24T07:35:27,941 DEBUG [StoreOpener-b25495aaf2c830ef51e5e99e180edf06-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:33505/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/b25495aaf2c830ef51e5e99e180edf06/a/dfa272b15dc7460195b1762575a439c1_SeqId_3_ 2024-11-24T07:35:27,941 INFO [StoreOpener-b25495aaf2c830ef51e5e99e180edf06-1 {}] regionserver.HStore(327): Store=b25495aaf2c830ef51e5e99e180edf06/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T07:35:27,942 INFO [StoreOpener-b25495aaf2c830ef51e5e99e180edf06-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region b25495aaf2c830ef51e5e99e180edf06 2024-11-24T07:35:27,943 INFO [StoreOpener-b25495aaf2c830ef51e5e99e180edf06-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region b25495aaf2c830ef51e5e99e180edf06 columnFamilyName b 2024-11-24T07:35:27,943 DEBUG [StoreOpener-b25495aaf2c830ef51e5e99e180edf06-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T07:35:27,943 INFO [StoreOpener-b25495aaf2c830ef51e5e99e180edf06-1 {}] regionserver.HStore(327): Store=b25495aaf2c830ef51e5e99e180edf06/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T07:35:27,943 INFO [StoreOpener-b25495aaf2c830ef51e5e99e180edf06-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region b25495aaf2c830ef51e5e99e180edf06 2024-11-24T07:35:27,944 INFO [StoreOpener-b25495aaf2c830ef51e5e99e180edf06-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region b25495aaf2c830ef51e5e99e180edf06 columnFamilyName c 2024-11-24T07:35:27,944 DEBUG [StoreOpener-b25495aaf2c830ef51e5e99e180edf06-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T07:35:27,945 INFO [StoreOpener-b25495aaf2c830ef51e5e99e180edf06-1 {}] regionserver.HStore(327): Store=b25495aaf2c830ef51e5e99e180edf06/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T07:35:27,945 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for b25495aaf2c830ef51e5e99e180edf06 2024-11-24T07:35:27,945 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33505/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/b25495aaf2c830ef51e5e99e180edf06 2024-11-24T07:35:27,947 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:33505/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/b25495aaf2c830ef51e5e99e180edf06 2024-11-24T07:35:27,947 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:33505/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/b25495aaf2c830ef51e5e99e180edf06/recovered.edits/0000000000000000005 2024-11-24T07:35:27,949 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:33505/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/b25495aaf2c830ef51e5e99e180edf06/recovered.edits/0000000000000000005: isRecoveredEdits=true, hasTagCompression=true, hasValueCompression=true, valueCompressionType=GZ 2024-11-24T07:35:27,950 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 1, skipped 0, firstSequenceIdInLog=5, maxSequenceIdInLog=5, path=hdfs://localhost:33505/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/b25495aaf2c830ef51e5e99e180edf06/recovered.edits/0000000000000000005 2024-11-24T07:35:27,950 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing b25495aaf2c830ef51e5e99e180edf06 3/3 column families, dataSize=58 B heapSize=904 B 2024-11-24T07:35:27,965 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33505/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/b25495aaf2c830ef51e5e99e180edf06/.tmp/a/5f9a064c0c704ff5bf8fbd08775820cf is 62, key is testRegionMadeOfBulkLoadedFilesOnly/a:a/1732433723836/Put/seqid=0 2024-11-24T07:35:27,971 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41971 is added to blk_1073741884_1062 (size=5149) 2024-11-24T07:35:27,971 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42685 is added to blk_1073741884_1062 (size=5149) 2024-11-24T07:35:27,971 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41161 is added to blk_1073741884_1062 (size=5149) 2024-11-24T07:35:27,972 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=58 B at sequenceid=5 (bloomFilter=true), to=hdfs://localhost:33505/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/b25495aaf2c830ef51e5e99e180edf06/.tmp/a/5f9a064c0c704ff5bf8fbd08775820cf 2024-11-24T07:35:27,977 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33505/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/b25495aaf2c830ef51e5e99e180edf06/.tmp/a/5f9a064c0c704ff5bf8fbd08775820cf as hdfs://localhost:33505/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/b25495aaf2c830ef51e5e99e180edf06/a/5f9a064c0c704ff5bf8fbd08775820cf 2024-11-24T07:35:27,983 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33505/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/b25495aaf2c830ef51e5e99e180edf06/a/5f9a064c0c704ff5bf8fbd08775820cf, entries=1, sequenceid=5, filesize=5.0 K 2024-11-24T07:35:27,983 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~58 B/58, heapSize ~376 B/376, currentSize=0 B/0 for b25495aaf2c830ef51e5e99e180edf06 in 33ms, sequenceid=5, compaction requested=false; wal=null 2024-11-24T07:35:27,984 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:33505/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/b25495aaf2c830ef51e5e99e180edf06/recovered.edits/0000000000000000005 2024-11-24T07:35:27,985 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for b25495aaf2c830ef51e5e99e180edf06 2024-11-24T07:35:27,986 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for b25495aaf2c830ef51e5e99e180edf06 2024-11-24T07:35:27,986 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testRegionMadeOfBulkLoadedFilesOnly descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-24T07:35:27,988 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for b25495aaf2c830ef51e5e99e180edf06 2024-11-24T07:35:27,991 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:33505/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/b25495aaf2c830ef51e5e99e180edf06/recovered.edits/5.seqid, newMaxSeqId=5, maxSeqId=1 2024-11-24T07:35:27,992 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened b25495aaf2c830ef51e5e99e180edf06; next sequenceid=6; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=61369928, jitterRate=-0.08551681041717529}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-24T07:35:27,992 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for b25495aaf2c830ef51e5e99e180edf06: Writing region info on filesystem at 1732433727931Initializing all the Stores at 1732433727932 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732433727932Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732433727932Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732433727932Obtaining lock to block concurrent updates at 1732433727950 (+18 ms)Preparing flush snapshotting stores in b25495aaf2c830ef51e5e99e180edf06 at 1732433727950Finished memstore snapshotting testRegionMadeOfBulkLoadedFilesOnly,,1732433723665.b25495aaf2c830ef51e5e99e180edf06., syncing WAL and waiting on mvcc, flushsize=dataSize=58, getHeapSize=856, getOffHeapSize=0, getCellsCount=1 at 1732433727950Flushing stores of testRegionMadeOfBulkLoadedFilesOnly,,1732433723665.b25495aaf2c830ef51e5e99e180edf06. at 1732433727950Flushing b25495aaf2c830ef51e5e99e180edf06/a: creating writer at 1732433727950Flushing b25495aaf2c830ef51e5e99e180edf06/a: appending metadata at 1732433727964 (+14 ms)Flushing b25495aaf2c830ef51e5e99e180edf06/a: closing flushed file at 1732433727965 (+1 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@395181dc: reopening flushed file at 1732433727976 (+11 ms)Finished flush of dataSize ~58 B/58, heapSize ~376 B/376, currentSize=0 B/0 for b25495aaf2c830ef51e5e99e180edf06 in 33ms, sequenceid=5, compaction requested=false; wal=null at 1732433727983 (+7 ms)Cleaning up temporary data from old regions at 1732433727986 (+3 ms)Region opened successfully at 1732433727992 (+6 ms) 2024-11-24T07:35:27,997 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing b25495aaf2c830ef51e5e99e180edf06, disabling compactions & flushes 2024-11-24T07:35:27,997 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testRegionMadeOfBulkLoadedFilesOnly,,1732433723665.b25495aaf2c830ef51e5e99e180edf06. 2024-11-24T07:35:27,997 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testRegionMadeOfBulkLoadedFilesOnly,,1732433723665.b25495aaf2c830ef51e5e99e180edf06. 2024-11-24T07:35:27,997 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testRegionMadeOfBulkLoadedFilesOnly,,1732433723665.b25495aaf2c830ef51e5e99e180edf06. after waiting 0 ms 2024-11-24T07:35:27,997 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testRegionMadeOfBulkLoadedFilesOnly,,1732433723665.b25495aaf2c830ef51e5e99e180edf06. 2024-11-24T07:35:27,998 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testRegionMadeOfBulkLoadedFilesOnly,,1732433723665.b25495aaf2c830ef51e5e99e180edf06. 2024-11-24T07:35:27,998 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for b25495aaf2c830ef51e5e99e180edf06: Waiting for close lock at 1732433727997Disabling compacts and flushes for region at 1732433727997Disabling writes for close at 1732433727997Writing region close event to WAL at 1732433727998 (+1 ms)Closed at 1732433727998 2024-11-24T07:35:28,001 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41971 is added to blk_1073741883_1061 (size=95) 2024-11-24T07:35:28,001 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41161 is added to blk_1073741883_1061 (size=95) 2024-11-24T07:35:28,002 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42685 is added to blk_1073741883_1061 (size=95) 2024-11-24T07:35:28,004 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-11-24T07:35:28,004 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL wal:(num 1732433727910) 2024-11-24T07:35:28,018 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestAsyncWALReplayValueCompression#testRegionMadeOfBulkLoadedFilesOnly Thread=412 (was 408) Potentially hanging thread: AsyncFSWAL-17-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-963511333_22 at /127.0.0.1:52922 [Waiting for operation #9] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/089ee269-8889-d2f6-f8a2-4bb8eac365fc/cluster_a7d76481-495d-c9f6-361f-c7016573445a/data/data1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Client (1319272049) connection to localhost/127.0.0.1:33505 from jenkinstestRegionMadeOfBulkLoadedFilesOnly java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-963511333_22 at /127.0.0.1:60130 [Waiting for operation #7] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-17-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/089ee269-8889-d2f6-f8a2-4bb8eac365fc/cluster_a7d76481-495d-c9f6-361f-c7016573445a/data/data2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-17-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: LeaseRenewer:jenkinstestRegionMadeOfBulkLoadedFilesOnly@localhost:33505 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=1024 (was 967) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=228 (was 247), ProcessCount=11 (was 11), AvailableMemoryMB=2185 (was 2177) - AvailableMemoryMB LEAK? - 2024-11-24T07:35:28,030 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestAsyncWALReplayValueCompression#testReplayEditsAfterRegionMovedWithMultiCF Thread=412, OpenFileDescriptor=1024, MaxFileDescriptor=1048576, SystemLoadAverage=228, ProcessCount=11, AvailableMemoryMB=2184 2024-11-24T07:35:28,046 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-24T07:35:28,050 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-24T07:35:28,054 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] client.AsyncConnectionImpl(321): The fetched master address is 1ab588ae992e,37719,1732433712713 2024-11-24T07:35:28,056 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@7b19545a 2024-11-24T07:35:28,056 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-24T07:35:28,058 INFO [HMaster-EventLoopGroup-2-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:36222, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-24T07:35:28,061 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37719 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.3 create 'testReplayEditsAfterRegionMovedWithMultiCF', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'cf1', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'cf2', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-24T07:35:28,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37719 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=testReplayEditsAfterRegionMovedWithMultiCF 2024-11-24T07:35:28,069 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=testReplayEditsAfterRegionMovedWithMultiCF execute state=CREATE_TABLE_PRE_OPERATION 2024-11-24T07:35:28,071 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37719 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.3 procedure request for creating table: namespace: "default" qualifier: "testReplayEditsAfterRegionMovedWithMultiCF" procId is: 4 2024-11-24T07:35:28,072 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T07:35:28,074 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=testReplayEditsAfterRegionMovedWithMultiCF execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-24T07:35:28,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37719 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-24T07:35:28,085 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41161 is added to blk_1073741885_1063 (size=694) 2024-11-24T07:35:28,085 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41971 is added to blk_1073741885_1063 (size=694) 2024-11-24T07:35:28,085 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42685 is added to blk_1073741885_1063 (size=694) 2024-11-24T07:35:28,088 INFO [RegionOpenAndInit-testReplayEditsAfterRegionMovedWithMultiCF-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 567d9cab2ff71509e601db9c0fa44478, NAME => 'testReplayEditsAfterRegionMovedWithMultiCF,,1732433728060.567d9cab2ff71509e601db9c0fa44478.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testReplayEditsAfterRegionMovedWithMultiCF', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'cf1', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'cf2', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:33505/user/jenkins/test-data/e37915a4-8313-e328-91fb-37d541e8b214 2024-11-24T07:35:28,096 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41971 is added to blk_1073741886_1064 (size=77) 2024-11-24T07:35:28,096 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42685 is added to blk_1073741886_1064 (size=77) 2024-11-24T07:35:28,096 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41161 is added to blk_1073741886_1064 (size=77) 2024-11-24T07:35:28,097 DEBUG [RegionOpenAndInit-testReplayEditsAfterRegionMovedWithMultiCF-pool-0 {}] regionserver.HRegion(898): Instantiated testReplayEditsAfterRegionMovedWithMultiCF,,1732433728060.567d9cab2ff71509e601db9c0fa44478.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T07:35:28,097 DEBUG [RegionOpenAndInit-testReplayEditsAfterRegionMovedWithMultiCF-pool-0 {}] regionserver.HRegion(1722): Closing 567d9cab2ff71509e601db9c0fa44478, disabling compactions & flushes 2024-11-24T07:35:28,097 INFO [RegionOpenAndInit-testReplayEditsAfterRegionMovedWithMultiCF-pool-0 {}] regionserver.HRegion(1755): Closing region testReplayEditsAfterRegionMovedWithMultiCF,,1732433728060.567d9cab2ff71509e601db9c0fa44478. 2024-11-24T07:35:28,097 DEBUG [RegionOpenAndInit-testReplayEditsAfterRegionMovedWithMultiCF-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1732433728060.567d9cab2ff71509e601db9c0fa44478. 2024-11-24T07:35:28,097 DEBUG [RegionOpenAndInit-testReplayEditsAfterRegionMovedWithMultiCF-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1732433728060.567d9cab2ff71509e601db9c0fa44478. after waiting 0 ms 2024-11-24T07:35:28,097 DEBUG [RegionOpenAndInit-testReplayEditsAfterRegionMovedWithMultiCF-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsAfterRegionMovedWithMultiCF,,1732433728060.567d9cab2ff71509e601db9c0fa44478. 2024-11-24T07:35:28,097 INFO [RegionOpenAndInit-testReplayEditsAfterRegionMovedWithMultiCF-pool-0 {}] regionserver.HRegion(1973): Closed testReplayEditsAfterRegionMovedWithMultiCF,,1732433728060.567d9cab2ff71509e601db9c0fa44478. 2024-11-24T07:35:28,097 DEBUG [RegionOpenAndInit-testReplayEditsAfterRegionMovedWithMultiCF-pool-0 {}] regionserver.HRegion(1676): Region close journal for 567d9cab2ff71509e601db9c0fa44478: Waiting for close lock at 1732433728097Disabling compacts and flushes for region at 1732433728097Disabling writes for close at 1732433728097Writing region close event to WAL at 1732433728097Closed at 1732433728097 2024-11-24T07:35:28,099 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=testReplayEditsAfterRegionMovedWithMultiCF execute state=CREATE_TABLE_ADD_TO_META 2024-11-24T07:35:28,105 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testReplayEditsAfterRegionMovedWithMultiCF,,1732433728060.567d9cab2ff71509e601db9c0fa44478.","families":{"info":[{"qualifier":"regioninfo","vlen":76,"tag":[],"timestamp":"1732433728099"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732433728099"}]},"ts":"1732433728099"} 2024-11-24T07:35:28,108 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-24T07:35:28,109 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=testReplayEditsAfterRegionMovedWithMultiCF execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-24T07:35:28,112 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testReplayEditsAfterRegionMovedWithMultiCF","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732433728109"}]},"ts":"1732433728109"} 2024-11-24T07:35:28,115 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=testReplayEditsAfterRegionMovedWithMultiCF, state=ENABLING in hbase:meta 2024-11-24T07:35:28,116 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(204): Hosts are {1ab588ae992e=0} racks are {/default-rack=0} 2024-11-24T07:35:28,117 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-11-24T07:35:28,117 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-11-24T07:35:28,117 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(303): server 2 has 0 regions 2024-11-24T07:35:28,117 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-11-24T07:35:28,117 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-11-24T07:35:28,117 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(310): server 2 is on host 0 2024-11-24T07:35:28,117 INFO [PEWorker-3 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-11-24T07:35:28,117 INFO [PEWorker-3 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-11-24T07:35:28,117 INFO [PEWorker-3 {}] balancer.BalancerClusterState(321): server 2 is on rack 0 2024-11-24T07:35:28,117 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-11-24T07:35:28,118 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=567d9cab2ff71509e601db9c0fa44478, ASSIGN}] 2024-11-24T07:35:28,120 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=567d9cab2ff71509e601db9c0fa44478, ASSIGN 2024-11-24T07:35:28,122 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=567d9cab2ff71509e601db9c0fa44478, ASSIGN; state=OFFLINE, location=1ab588ae992e,40611,1732433713633; forceNewPlan=false, retain=false 2024-11-24T07:35:28,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37719 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-24T07:35:28,278 INFO [1ab588ae992e:37719 {}] balancer.BaseLoadBalancer(388): Reassigned 1 regions. 1 retained the pre-restart assignment. 2024-11-24T07:35:28,280 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=567d9cab2ff71509e601db9c0fa44478, regionState=OPENING, regionLocation=1ab588ae992e,40611,1732433713633 2024-11-24T07:35:28,283 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=567d9cab2ff71509e601db9c0fa44478, ASSIGN because future has completed 2024-11-24T07:35:28,284 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 567d9cab2ff71509e601db9c0fa44478, server=1ab588ae992e,40611,1732433713633}] 2024-11-24T07:35:28,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37719 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-24T07:35:28,445 INFO [RS_OPEN_REGION-regionserver/1ab588ae992e:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(132): Open testReplayEditsAfterRegionMovedWithMultiCF,,1732433728060.567d9cab2ff71509e601db9c0fa44478. 2024-11-24T07:35:28,446 DEBUG [RS_OPEN_REGION-regionserver/1ab588ae992e:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7752): Opening region: {ENCODED => 567d9cab2ff71509e601db9c0fa44478, NAME => 'testReplayEditsAfterRegionMovedWithMultiCF,,1732433728060.567d9cab2ff71509e601db9c0fa44478.', STARTKEY => '', ENDKEY => ''} 2024-11-24T07:35:28,446 DEBUG [RS_OPEN_REGION-regionserver/1ab588ae992e:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testReplayEditsAfterRegionMovedWithMultiCF 567d9cab2ff71509e601db9c0fa44478 2024-11-24T07:35:28,446 DEBUG [RS_OPEN_REGION-regionserver/1ab588ae992e:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(898): Instantiated testReplayEditsAfterRegionMovedWithMultiCF,,1732433728060.567d9cab2ff71509e601db9c0fa44478.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T07:35:28,446 DEBUG [RS_OPEN_REGION-regionserver/1ab588ae992e:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7794): checking encryption for 567d9cab2ff71509e601db9c0fa44478 2024-11-24T07:35:28,446 DEBUG [RS_OPEN_REGION-regionserver/1ab588ae992e:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7797): checking classloading for 567d9cab2ff71509e601db9c0fa44478 2024-11-24T07:35:28,450 INFO [StoreOpener-567d9cab2ff71509e601db9c0fa44478-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf1 of region 567d9cab2ff71509e601db9c0fa44478 2024-11-24T07:35:28,451 INFO [StoreOpener-567d9cab2ff71509e601db9c0fa44478-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 567d9cab2ff71509e601db9c0fa44478 columnFamilyName cf1 2024-11-24T07:35:28,451 DEBUG [StoreOpener-567d9cab2ff71509e601db9c0fa44478-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T07:35:28,452 INFO [StoreOpener-567d9cab2ff71509e601db9c0fa44478-1 {}] regionserver.HStore(327): Store=567d9cab2ff71509e601db9c0fa44478/cf1, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T07:35:28,452 INFO [StoreOpener-567d9cab2ff71509e601db9c0fa44478-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf2 of region 567d9cab2ff71509e601db9c0fa44478 2024-11-24T07:35:28,453 INFO [StoreOpener-567d9cab2ff71509e601db9c0fa44478-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 567d9cab2ff71509e601db9c0fa44478 columnFamilyName cf2 2024-11-24T07:35:28,454 DEBUG [StoreOpener-567d9cab2ff71509e601db9c0fa44478-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T07:35:28,454 INFO [StoreOpener-567d9cab2ff71509e601db9c0fa44478-1 {}] regionserver.HStore(327): Store=567d9cab2ff71509e601db9c0fa44478/cf2, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T07:35:28,454 DEBUG [RS_OPEN_REGION-regionserver/1ab588ae992e:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1038): replaying wal for 567d9cab2ff71509e601db9c0fa44478 2024-11-24T07:35:28,455 DEBUG [RS_OPEN_REGION-regionserver/1ab588ae992e:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33505/user/jenkins/test-data/e37915a4-8313-e328-91fb-37d541e8b214/data/default/testReplayEditsAfterRegionMovedWithMultiCF/567d9cab2ff71509e601db9c0fa44478 2024-11-24T07:35:28,455 DEBUG [RS_OPEN_REGION-regionserver/1ab588ae992e:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33505/user/jenkins/test-data/e37915a4-8313-e328-91fb-37d541e8b214/data/default/testReplayEditsAfterRegionMovedWithMultiCF/567d9cab2ff71509e601db9c0fa44478 2024-11-24T07:35:28,456 DEBUG [RS_OPEN_REGION-regionserver/1ab588ae992e:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1048): stopping wal replay for 567d9cab2ff71509e601db9c0fa44478 2024-11-24T07:35:28,456 DEBUG [RS_OPEN_REGION-regionserver/1ab588ae992e:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1060): Cleaning up temporary data for 567d9cab2ff71509e601db9c0fa44478 2024-11-24T07:35:28,457 DEBUG [RS_OPEN_REGION-regionserver/1ab588ae992e:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsAfterRegionMovedWithMultiCF descriptor;using region.getMemStoreFlushHeapSize/# of families (64.0 M)) instead. 2024-11-24T07:35:28,458 DEBUG [RS_OPEN_REGION-regionserver/1ab588ae992e:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1093): writing seq id for 567d9cab2ff71509e601db9c0fa44478 2024-11-24T07:35:28,461 DEBUG [RS_OPEN_REGION-regionserver/1ab588ae992e:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:33505/user/jenkins/test-data/e37915a4-8313-e328-91fb-37d541e8b214/data/default/testReplayEditsAfterRegionMovedWithMultiCF/567d9cab2ff71509e601db9c0fa44478/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-24T07:35:28,461 INFO [RS_OPEN_REGION-regionserver/1ab588ae992e:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1114): Opened 567d9cab2ff71509e601db9c0fa44478; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=67201443, jitterRate=0.0013795346021652222}}}, FlushLargeStoresPolicy{flushSizeLowerBound=67108864} 2024-11-24T07:35:28,462 DEBUG [RS_OPEN_REGION-regionserver/1ab588ae992e:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 567d9cab2ff71509e601db9c0fa44478 2024-11-24T07:35:28,462 DEBUG [RS_OPEN_REGION-regionserver/1ab588ae992e:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1006): Region open journal for 567d9cab2ff71509e601db9c0fa44478: Running coprocessor pre-open hook at 1732433728447Writing region info on filesystem at 1732433728447Initializing all the Stores at 1732433728448 (+1 ms)Instantiating store for column family {NAME => 'cf1', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732433728448Instantiating store for column family {NAME => 'cf2', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732433728449 (+1 ms)Cleaning up temporary data from old regions at 1732433728456 (+7 ms)Running coprocessor post-open hooks at 1732433728462 (+6 ms)Region opened successfully at 1732433728462 2024-11-24T07:35:28,464 INFO [RS_OPEN_REGION-regionserver/1ab588ae992e:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2236): Post open deploy tasks for testReplayEditsAfterRegionMovedWithMultiCF,,1732433728060.567d9cab2ff71509e601db9c0fa44478., pid=6, masterSystemTime=1732433728438 2024-11-24T07:35:28,468 DEBUG [RS_OPEN_REGION-regionserver/1ab588ae992e:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2266): Finished post open deploy task for testReplayEditsAfterRegionMovedWithMultiCF,,1732433728060.567d9cab2ff71509e601db9c0fa44478. 2024-11-24T07:35:28,468 INFO [RS_OPEN_REGION-regionserver/1ab588ae992e:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(153): Opened testReplayEditsAfterRegionMovedWithMultiCF,,1732433728060.567d9cab2ff71509e601db9c0fa44478. 2024-11-24T07:35:28,469 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=567d9cab2ff71509e601db9c0fa44478, regionState=OPEN, openSeqNum=2, regionLocation=1ab588ae992e,40611,1732433713633 2024-11-24T07:35:28,472 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 567d9cab2ff71509e601db9c0fa44478, server=1ab588ae992e,40611,1732433713633 because future has completed 2024-11-24T07:35:28,478 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=6, resume processing ppid=5 2024-11-24T07:35:28,478 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure 567d9cab2ff71509e601db9c0fa44478, server=1ab588ae992e,40611,1732433713633 in 190 msec 2024-11-24T07:35:28,482 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=5, resume processing ppid=4 2024-11-24T07:35:28,482 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=567d9cab2ff71509e601db9c0fa44478, ASSIGN in 360 msec 2024-11-24T07:35:28,483 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=testReplayEditsAfterRegionMovedWithMultiCF execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-24T07:35:28,483 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testReplayEditsAfterRegionMovedWithMultiCF","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732433728483"}]},"ts":"1732433728483"} 2024-11-24T07:35:28,486 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=testReplayEditsAfterRegionMovedWithMultiCF, state=ENABLED in hbase:meta 2024-11-24T07:35:28,487 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=testReplayEditsAfterRegionMovedWithMultiCF execute state=CREATE_TABLE_POST_OPERATION 2024-11-24T07:35:28,489 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=testReplayEditsAfterRegionMovedWithMultiCF in 424 msec 2024-11-24T07:35:28,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37719 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-24T07:35:28,706 INFO [RPCClient-NioEventLoopGroup-6-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:testReplayEditsAfterRegionMovedWithMultiCF completed 2024-11-24T07:35:28,706 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table testReplayEditsAfterRegionMovedWithMultiCF get assigned. Timeout = 60000ms 2024-11-24T07:35:28,709 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-24T07:35:28,720 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table testReplayEditsAfterRegionMovedWithMultiCF assigned to meta. Checking AM states. 2024-11-24T07:35:28,720 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-24T07:35:28,721 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table testReplayEditsAfterRegionMovedWithMultiCF assigned. 2024-11-24T07:35:28,737 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testReplayEditsAfterRegionMovedWithMultiCF', row='r1', locateType=CURRENT is [region=testReplayEditsAfterRegionMovedWithMultiCF,,1732433728060.567d9cab2ff71509e601db9c0fa44478., hostname=1ab588ae992e,40611,1732433713633, seqNum=2] 2024-11-24T07:35:28,752 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37719 {}] master.HMaster(2410): Client=jenkins//172.17.0.3 move hri=567d9cab2ff71509e601db9c0fa44478, source=1ab588ae992e,40611,1732433713633, destination=1ab588ae992e,46433,1732433713473, warming up region on 1ab588ae992e,46433,1732433713473 2024-11-24T07:35:28,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37719 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-24T07:35:28,756 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37719 {}] master.HMaster(2414): Client=jenkins//172.17.0.3 move hri=567d9cab2ff71509e601db9c0fa44478, source=1ab588ae992e,40611,1732433713633, destination=1ab588ae992e,46433,1732433713473, running balancer 2024-11-24T07:35:28,757 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:44627, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-24T07:35:28,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37719 {}] procedure2.ProcedureExecutor(1139): Stored pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=567d9cab2ff71509e601db9c0fa44478, REOPEN/MOVE 2024-11-24T07:35:28,758 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=567d9cab2ff71509e601db9c0fa44478, REOPEN/MOVE 2024-11-24T07:35:28,761 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=567d9cab2ff71509e601db9c0fa44478, regionState=CLOSING, regionLocation=1ab588ae992e,40611,1732433713633 2024-11-24T07:35:28,762 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46433 {}] regionserver.RSRpcServices(2066): Warmup testReplayEditsAfterRegionMovedWithMultiCF,,1732433728060.567d9cab2ff71509e601db9c0fa44478. 2024-11-24T07:35:28,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46433 {}] regionserver.HRegion(7855): Warmup {ENCODED => 567d9cab2ff71509e601db9c0fa44478, NAME => 'testReplayEditsAfterRegionMovedWithMultiCF,,1732433728060.567d9cab2ff71509e601db9c0fa44478.', STARTKEY => '', ENDKEY => ''} 2024-11-24T07:35:28,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46433 {}] regionserver.HRegion(898): Instantiated testReplayEditsAfterRegionMovedWithMultiCF,,1732433728060.567d9cab2ff71509e601db9c0fa44478.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T07:35:28,763 INFO [StoreOpener-567d9cab2ff71509e601db9c0fa44478-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf1 of region 567d9cab2ff71509e601db9c0fa44478 2024-11-24T07:35:28,765 INFO [StoreOpener-567d9cab2ff71509e601db9c0fa44478-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 567d9cab2ff71509e601db9c0fa44478 columnFamilyName cf1 2024-11-24T07:35:28,765 DEBUG [StoreOpener-567d9cab2ff71509e601db9c0fa44478-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T07:35:28,765 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=567d9cab2ff71509e601db9c0fa44478, REOPEN/MOVE because future has completed 2024-11-24T07:35:28,766 DEBUG [PEWorker-3 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-24T07:35:28,766 INFO [StoreOpener-567d9cab2ff71509e601db9c0fa44478-1 {}] regionserver.HStore(327): Store=567d9cab2ff71509e601db9c0fa44478/cf1, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T07:35:28,766 INFO [StoreOpener-567d9cab2ff71509e601db9c0fa44478-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf2 of region 567d9cab2ff71509e601db9c0fa44478 2024-11-24T07:35:28,766 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=8, ppid=7, state=RUNNABLE, hasLock=false; CloseRegionProcedure 567d9cab2ff71509e601db9c0fa44478, server=1ab588ae992e,40611,1732433713633}] 2024-11-24T07:35:28,767 INFO [StoreOpener-567d9cab2ff71509e601db9c0fa44478-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 567d9cab2ff71509e601db9c0fa44478 columnFamilyName cf2 2024-11-24T07:35:28,767 DEBUG [StoreOpener-567d9cab2ff71509e601db9c0fa44478-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T07:35:28,768 INFO [StoreOpener-567d9cab2ff71509e601db9c0fa44478-1 {}] regionserver.HStore(327): Store=567d9cab2ff71509e601db9c0fa44478/cf2, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T07:35:28,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46433 {}] regionserver.HRegion(1722): Closing 567d9cab2ff71509e601db9c0fa44478, disabling compactions & flushes 2024-11-24T07:35:28,768 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46433 {}] regionserver.HRegion(1755): Closing region testReplayEditsAfterRegionMovedWithMultiCF,,1732433728060.567d9cab2ff71509e601db9c0fa44478. 2024-11-24T07:35:28,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46433 {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1732433728060.567d9cab2ff71509e601db9c0fa44478. 2024-11-24T07:35:28,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46433 {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1732433728060.567d9cab2ff71509e601db9c0fa44478. after waiting 0 ms 2024-11-24T07:35:28,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46433 {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsAfterRegionMovedWithMultiCF,,1732433728060.567d9cab2ff71509e601db9c0fa44478. 2024-11-24T07:35:28,770 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46433 {}] regionserver.HRegion(1973): Closed testReplayEditsAfterRegionMovedWithMultiCF,,1732433728060.567d9cab2ff71509e601db9c0fa44478. 2024-11-24T07:35:28,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46433 {}] regionserver.HRegion(1676): Region close journal for 567d9cab2ff71509e601db9c0fa44478: Waiting for close lock at 1732433728768Disabling compacts and flushes for region at 1732433728768Disabling writes for close at 1732433728768Writing region close event to WAL at 1732433728770 (+2 ms)Closed at 1732433728770 2024-11-24T07:35:28,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37719 {}] procedure.ProcedureSyncWait(219): waitFor pid=7 2024-11-24T07:35:28,928 INFO [RS_CLOSE_REGION-regionserver/1ab588ae992e:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] handler.UnassignRegionHandler(122): Close 567d9cab2ff71509e601db9c0fa44478 2024-11-24T07:35:28,929 DEBUG [RS_CLOSE_REGION-regionserver/1ab588ae992e:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-24T07:35:28,930 DEBUG [RS_CLOSE_REGION-regionserver/1ab588ae992e:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1722): Closing 567d9cab2ff71509e601db9c0fa44478, disabling compactions & flushes 2024-11-24T07:35:28,930 INFO [RS_CLOSE_REGION-regionserver/1ab588ae992e:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1755): Closing region testReplayEditsAfterRegionMovedWithMultiCF,,1732433728060.567d9cab2ff71509e601db9c0fa44478. 2024-11-24T07:35:28,931 DEBUG [RS_CLOSE_REGION-regionserver/1ab588ae992e:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1732433728060.567d9cab2ff71509e601db9c0fa44478. 2024-11-24T07:35:28,931 DEBUG [RS_CLOSE_REGION-regionserver/1ab588ae992e:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1732433728060.567d9cab2ff71509e601db9c0fa44478. after waiting 0 ms 2024-11-24T07:35:28,931 DEBUG [RS_CLOSE_REGION-regionserver/1ab588ae992e:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsAfterRegionMovedWithMultiCF,,1732433728060.567d9cab2ff71509e601db9c0fa44478. 2024-11-24T07:35:28,931 INFO [RS_CLOSE_REGION-regionserver/1ab588ae992e:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(2902): Flushing 567d9cab2ff71509e601db9c0fa44478 2/2 column families, dataSize=31 B heapSize=616 B 2024-11-24T07:35:28,950 DEBUG [RS_CLOSE_REGION-regionserver/1ab588ae992e:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33505/user/jenkins/test-data/e37915a4-8313-e328-91fb-37d541e8b214/data/default/testReplayEditsAfterRegionMovedWithMultiCF/567d9cab2ff71509e601db9c0fa44478/.tmp/cf1/d657c95ee1b14d3e82ed05bfbe9a2cfb is 35, key is r1/cf1:q/1732433728739/Put/seqid=0 2024-11-24T07:35:28,962 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41971 is added to blk_1073741887_1065 (size=4783) 2024-11-24T07:35:28,963 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42685 is added to blk_1073741887_1065 (size=4783) 2024-11-24T07:35:28,963 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41161 is added to blk_1073741887_1065 (size=4783) 2024-11-24T07:35:28,965 INFO [RS_CLOSE_REGION-regionserver/1ab588ae992e:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31 B at sequenceid=5 (bloomFilter=false), to=hdfs://localhost:33505/user/jenkins/test-data/e37915a4-8313-e328-91fb-37d541e8b214/data/default/testReplayEditsAfterRegionMovedWithMultiCF/567d9cab2ff71509e601db9c0fa44478/.tmp/cf1/d657c95ee1b14d3e82ed05bfbe9a2cfb 2024-11-24T07:35:28,974 DEBUG [RS_CLOSE_REGION-regionserver/1ab588ae992e:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33505/user/jenkins/test-data/e37915a4-8313-e328-91fb-37d541e8b214/data/default/testReplayEditsAfterRegionMovedWithMultiCF/567d9cab2ff71509e601db9c0fa44478/.tmp/cf1/d657c95ee1b14d3e82ed05bfbe9a2cfb as hdfs://localhost:33505/user/jenkins/test-data/e37915a4-8313-e328-91fb-37d541e8b214/data/default/testReplayEditsAfterRegionMovedWithMultiCF/567d9cab2ff71509e601db9c0fa44478/cf1/d657c95ee1b14d3e82ed05bfbe9a2cfb 2024-11-24T07:35:28,982 INFO [RS_CLOSE_REGION-regionserver/1ab588ae992e:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33505/user/jenkins/test-data/e37915a4-8313-e328-91fb-37d541e8b214/data/default/testReplayEditsAfterRegionMovedWithMultiCF/567d9cab2ff71509e601db9c0fa44478/cf1/d657c95ee1b14d3e82ed05bfbe9a2cfb, entries=1, sequenceid=5, filesize=4.7 K 2024-11-24T07:35:28,983 INFO [RS_CLOSE_REGION-regionserver/1ab588ae992e:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(3140): Finished flush of dataSize ~31 B/31, heapSize ~344 B/344, currentSize=0 B/0 for 567d9cab2ff71509e601db9c0fa44478 in 52ms, sequenceid=5, compaction requested=false 2024-11-24T07:35:28,983 DEBUG [RS_CLOSE_REGION-regionserver/1ab588ae992e:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'testReplayEditsAfterRegionMovedWithMultiCF' 2024-11-24T07:35:28,990 DEBUG [RS_CLOSE_REGION-regionserver/1ab588ae992e:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:33505/user/jenkins/test-data/e37915a4-8313-e328-91fb-37d541e8b214/data/default/testReplayEditsAfterRegionMovedWithMultiCF/567d9cab2ff71509e601db9c0fa44478/recovered.edits/8.seqid, newMaxSeqId=8, maxSeqId=1 2024-11-24T07:35:28,993 INFO [RS_CLOSE_REGION-regionserver/1ab588ae992e:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1973): Closed testReplayEditsAfterRegionMovedWithMultiCF,,1732433728060.567d9cab2ff71509e601db9c0fa44478. 2024-11-24T07:35:28,993 DEBUG [RS_CLOSE_REGION-regionserver/1ab588ae992e:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1676): Region close journal for 567d9cab2ff71509e601db9c0fa44478: Waiting for close lock at 1732433728930Running coprocessor pre-close hooks at 1732433728930Disabling compacts and flushes for region at 1732433728930Disabling writes for close at 1732433728931 (+1 ms)Obtaining lock to block concurrent updates at 1732433728932 (+1 ms)Preparing flush snapshotting stores in 567d9cab2ff71509e601db9c0fa44478 at 1732433728932Finished memstore snapshotting testReplayEditsAfterRegionMovedWithMultiCF,,1732433728060.567d9cab2ff71509e601db9c0fa44478., syncing WAL and waiting on mvcc, flushsize=dataSize=31, getHeapSize=584, getOffHeapSize=0, getCellsCount=1 at 1732433728932Flushing stores of testReplayEditsAfterRegionMovedWithMultiCF,,1732433728060.567d9cab2ff71509e601db9c0fa44478. at 1732433728934 (+2 ms)Flushing 567d9cab2ff71509e601db9c0fa44478/cf1: creating writer at 1732433728934Flushing 567d9cab2ff71509e601db9c0fa44478/cf1: appending metadata at 1732433728950 (+16 ms)Flushing 567d9cab2ff71509e601db9c0fa44478/cf1: closing flushed file at 1732433728950Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@5c395e57: reopening flushed file at 1732433728973 (+23 ms)Finished flush of dataSize ~31 B/31, heapSize ~344 B/344, currentSize=0 B/0 for 567d9cab2ff71509e601db9c0fa44478 in 52ms, sequenceid=5, compaction requested=false at 1732433728983 (+10 ms)Writing region close event to WAL at 1732433728985 (+2 ms)Running coprocessor post-close hooks at 1732433728991 (+6 ms)Closed at 1732433728993 (+2 ms) 2024-11-24T07:35:28,994 INFO [RS_CLOSE_REGION-regionserver/1ab588ae992e:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegionServer(3302): Adding 567d9cab2ff71509e601db9c0fa44478 move to 1ab588ae992e,46433,1732433713473 record at close sequenceid=5 2024-11-24T07:35:29,001 INFO [RS_CLOSE_REGION-regionserver/1ab588ae992e:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] handler.UnassignRegionHandler(157): Closed 567d9cab2ff71509e601db9c0fa44478 2024-11-24T07:35:29,002 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=567d9cab2ff71509e601db9c0fa44478, regionState=CLOSED 2024-11-24T07:35:29,007 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=8, ppid=7, state=RUNNABLE, hasLock=false; CloseRegionProcedure 567d9cab2ff71509e601db9c0fa44478, server=1ab588ae992e,40611,1732433713633 because future has completed 2024-11-24T07:35:29,014 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=8, resume processing ppid=7 2024-11-24T07:35:29,014 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=8, ppid=7, state=SUCCESS, hasLock=false; CloseRegionProcedure 567d9cab2ff71509e601db9c0fa44478, server=1ab588ae992e,40611,1732433713633 in 243 msec 2024-11-24T07:35:29,015 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=567d9cab2ff71509e601db9c0fa44478, REOPEN/MOVE; state=CLOSED, location=1ab588ae992e,46433,1732433713473; forceNewPlan=false, retain=false 2024-11-24T07:35:29,165 INFO [1ab588ae992e:37719 {}] balancer.BaseLoadBalancer(388): Reassigned 1 regions. 1 retained the pre-restart assignment. 2024-11-24T07:35:29,166 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=567d9cab2ff71509e601db9c0fa44478, regionState=OPENING, regionLocation=1ab588ae992e,46433,1732433713473 2024-11-24T07:35:29,169 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=567d9cab2ff71509e601db9c0fa44478, REOPEN/MOVE because future has completed 2024-11-24T07:35:29,170 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=9, ppid=7, state=RUNNABLE, hasLock=false; OpenRegionProcedure 567d9cab2ff71509e601db9c0fa44478, server=1ab588ae992e,46433,1732433713473}] 2024-11-24T07:35:29,328 INFO [RS_OPEN_REGION-regionserver/1ab588ae992e:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] handler.AssignRegionHandler(132): Open testReplayEditsAfterRegionMovedWithMultiCF,,1732433728060.567d9cab2ff71509e601db9c0fa44478. 2024-11-24T07:35:29,329 DEBUG [RS_OPEN_REGION-regionserver/1ab588ae992e:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(7752): Opening region: {ENCODED => 567d9cab2ff71509e601db9c0fa44478, NAME => 'testReplayEditsAfterRegionMovedWithMultiCF,,1732433728060.567d9cab2ff71509e601db9c0fa44478.', STARTKEY => '', ENDKEY => ''} 2024-11-24T07:35:29,329 DEBUG [RS_OPEN_REGION-regionserver/1ab588ae992e:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testReplayEditsAfterRegionMovedWithMultiCF 567d9cab2ff71509e601db9c0fa44478 2024-11-24T07:35:29,329 DEBUG [RS_OPEN_REGION-regionserver/1ab588ae992e:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(898): Instantiated testReplayEditsAfterRegionMovedWithMultiCF,,1732433728060.567d9cab2ff71509e601db9c0fa44478.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T07:35:29,329 DEBUG [RS_OPEN_REGION-regionserver/1ab588ae992e:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(7794): checking encryption for 567d9cab2ff71509e601db9c0fa44478 2024-11-24T07:35:29,329 DEBUG [RS_OPEN_REGION-regionserver/1ab588ae992e:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(7797): checking classloading for 567d9cab2ff71509e601db9c0fa44478 2024-11-24T07:35:29,331 INFO [StoreOpener-567d9cab2ff71509e601db9c0fa44478-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf1 of region 567d9cab2ff71509e601db9c0fa44478 2024-11-24T07:35:29,332 INFO [StoreOpener-567d9cab2ff71509e601db9c0fa44478-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 567d9cab2ff71509e601db9c0fa44478 columnFamilyName cf1 2024-11-24T07:35:29,332 DEBUG [StoreOpener-567d9cab2ff71509e601db9c0fa44478-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T07:35:29,340 DEBUG [StoreOpener-567d9cab2ff71509e601db9c0fa44478-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:33505/user/jenkins/test-data/e37915a4-8313-e328-91fb-37d541e8b214/data/default/testReplayEditsAfterRegionMovedWithMultiCF/567d9cab2ff71509e601db9c0fa44478/cf1/d657c95ee1b14d3e82ed05bfbe9a2cfb 2024-11-24T07:35:29,340 INFO [StoreOpener-567d9cab2ff71509e601db9c0fa44478-1 {}] regionserver.HStore(327): Store=567d9cab2ff71509e601db9c0fa44478/cf1, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T07:35:29,340 INFO [StoreOpener-567d9cab2ff71509e601db9c0fa44478-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf2 of region 567d9cab2ff71509e601db9c0fa44478 2024-11-24T07:35:29,341 INFO [StoreOpener-567d9cab2ff71509e601db9c0fa44478-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 567d9cab2ff71509e601db9c0fa44478 columnFamilyName cf2 2024-11-24T07:35:29,341 DEBUG [StoreOpener-567d9cab2ff71509e601db9c0fa44478-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T07:35:29,342 INFO [StoreOpener-567d9cab2ff71509e601db9c0fa44478-1 {}] regionserver.HStore(327): Store=567d9cab2ff71509e601db9c0fa44478/cf2, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T07:35:29,342 DEBUG [RS_OPEN_REGION-regionserver/1ab588ae992e:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1038): replaying wal for 567d9cab2ff71509e601db9c0fa44478 2024-11-24T07:35:29,343 DEBUG [RS_OPEN_REGION-regionserver/1ab588ae992e:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33505/user/jenkins/test-data/e37915a4-8313-e328-91fb-37d541e8b214/data/default/testReplayEditsAfterRegionMovedWithMultiCF/567d9cab2ff71509e601db9c0fa44478 2024-11-24T07:35:29,344 DEBUG [RS_OPEN_REGION-regionserver/1ab588ae992e:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33505/user/jenkins/test-data/e37915a4-8313-e328-91fb-37d541e8b214/data/default/testReplayEditsAfterRegionMovedWithMultiCF/567d9cab2ff71509e601db9c0fa44478 2024-11-24T07:35:29,345 DEBUG [RS_OPEN_REGION-regionserver/1ab588ae992e:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1048): stopping wal replay for 567d9cab2ff71509e601db9c0fa44478 2024-11-24T07:35:29,345 DEBUG [RS_OPEN_REGION-regionserver/1ab588ae992e:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1060): Cleaning up temporary data for 567d9cab2ff71509e601db9c0fa44478 2024-11-24T07:35:29,346 DEBUG [RS_OPEN_REGION-regionserver/1ab588ae992e:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsAfterRegionMovedWithMultiCF descriptor;using region.getMemStoreFlushHeapSize/# of families (64.0 M)) instead. 2024-11-24T07:35:29,347 DEBUG [RS_OPEN_REGION-regionserver/1ab588ae992e:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1093): writing seq id for 567d9cab2ff71509e601db9c0fa44478 2024-11-24T07:35:29,348 INFO [RS_OPEN_REGION-regionserver/1ab588ae992e:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1114): Opened 567d9cab2ff71509e601db9c0fa44478; next sequenceid=9; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=61590178, jitterRate=-0.08223482966423035}}}, FlushLargeStoresPolicy{flushSizeLowerBound=67108864} 2024-11-24T07:35:29,348 DEBUG [RS_OPEN_REGION-regionserver/1ab588ae992e:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 567d9cab2ff71509e601db9c0fa44478 2024-11-24T07:35:29,349 DEBUG [RS_OPEN_REGION-regionserver/1ab588ae992e:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1006): Region open journal for 567d9cab2ff71509e601db9c0fa44478: Running coprocessor pre-open hook at 1732433729329Writing region info on filesystem at 1732433729329Initializing all the Stores at 1732433729331 (+2 ms)Instantiating store for column family {NAME => 'cf1', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732433729331Instantiating store for column family {NAME => 'cf2', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732433729331Cleaning up temporary data from old regions at 1732433729345 (+14 ms)Running coprocessor post-open hooks at 1732433729349 (+4 ms)Region opened successfully at 1732433729349 2024-11-24T07:35:29,350 INFO [RS_OPEN_REGION-regionserver/1ab588ae992e:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegionServer(2236): Post open deploy tasks for testReplayEditsAfterRegionMovedWithMultiCF,,1732433728060.567d9cab2ff71509e601db9c0fa44478., pid=9, masterSystemTime=1732433729324 2024-11-24T07:35:29,353 DEBUG [RS_OPEN_REGION-regionserver/1ab588ae992e:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegionServer(2266): Finished post open deploy task for testReplayEditsAfterRegionMovedWithMultiCF,,1732433728060.567d9cab2ff71509e601db9c0fa44478. 2024-11-24T07:35:29,353 INFO [RS_OPEN_REGION-regionserver/1ab588ae992e:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] handler.AssignRegionHandler(153): Opened testReplayEditsAfterRegionMovedWithMultiCF,,1732433728060.567d9cab2ff71509e601db9c0fa44478. 2024-11-24T07:35:29,354 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=567d9cab2ff71509e601db9c0fa44478, regionState=OPEN, openSeqNum=9, regionLocation=1ab588ae992e,46433,1732433713473 2024-11-24T07:35:29,357 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=9, ppid=7, state=RUNNABLE, hasLock=false; OpenRegionProcedure 567d9cab2ff71509e601db9c0fa44478, server=1ab588ae992e,46433,1732433713473 because future has completed 2024-11-24T07:35:29,362 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=9, resume processing ppid=7 2024-11-24T07:35:29,362 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=9, ppid=7, state=SUCCESS, hasLock=false; OpenRegionProcedure 567d9cab2ff71509e601db9c0fa44478, server=1ab588ae992e,46433,1732433713473 in 189 msec 2024-11-24T07:35:29,366 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=7, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=567d9cab2ff71509e601db9c0fa44478, REOPEN/MOVE in 607 msec 2024-11-24T07:35:29,414 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-24T07:35:29,417 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:41966, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-24T07:35:29,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40611 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 139 connection: 172.17.0.3:38364 deadline: 1732433789421, exception=org.apache.hadoop.hbase.exceptions.RegionMovedException: Region moved to: hostname=1ab588ae992e port=46433 startCode=1732433713473. As of locationSeqNum=5. 2024-11-24T07:35:29,446 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1732433728060.567d9cab2ff71509e601db9c0fa44478., hostname=1ab588ae992e,40611,1732433713633, seqNum=2 , the old value is region=testReplayEditsAfterRegionMovedWithMultiCF,,1732433728060.567d9cab2ff71509e601db9c0fa44478., hostname=1ab588ae992e,40611,1732433713633, seqNum=2, error=org.apache.hadoop.hbase.exceptions.RegionMovedException: Region moved to: hostname=1ab588ae992e port=46433 startCode=1732433713473. As of locationSeqNum=5. 2024-11-24T07:35:29,447 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1732433728060.567d9cab2ff71509e601db9c0fa44478., hostname=1ab588ae992e,40611,1732433713633, seqNum=2 is org.apache.hadoop.hbase.exceptions.RegionMovedException: Region moved to: hostname=1ab588ae992e port=46433 startCode=1732433713473. As of locationSeqNum=5. 2024-11-24T07:35:29,447 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncRegionLocatorHelper(84): Try updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1732433728060.567d9cab2ff71509e601db9c0fa44478., hostname=1ab588ae992e,40611,1732433713633, seqNum=2 with the new location region=testReplayEditsAfterRegionMovedWithMultiCF,,1732433728060.567d9cab2ff71509e601db9c0fa44478., hostname=1ab588ae992e,46433,1732433713473, seqNum=5 constructed by org.apache.hadoop.hbase.exceptions.RegionMovedException: Region moved to: hostname=1ab588ae992e port=46433 startCode=1732433713473. As of locationSeqNum=5. 2024-11-24T07:35:29,553 DEBUG [Async-Client-Retry-Timer-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-24T07:35:29,555 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:41970, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-24T07:35:29,564 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 567d9cab2ff71509e601db9c0fa44478 2/2 column families, dataSize=50 B heapSize=720 B 2024-11-24T07:35:29,584 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33505/user/jenkins/test-data/e37915a4-8313-e328-91fb-37d541e8b214/data/default/testReplayEditsAfterRegionMovedWithMultiCF/567d9cab2ff71509e601db9c0fa44478/.tmp/cf1/b885b098c54744cd9d0d8b9ad1b63b3d is 29, key is r1/cf1:/1732433729556/DeleteFamily/seqid=0 2024-11-24T07:35:29,591 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41161 is added to blk_1073741888_1066 (size=4906) 2024-11-24T07:35:29,591 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41971 is added to blk_1073741888_1066 (size=4906) 2024-11-24T07:35:29,591 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42685 is added to blk_1073741888_1066 (size=4906) 2024-11-24T07:35:29,591 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=25 B at sequenceid=12 (bloomFilter=false), to=hdfs://localhost:33505/user/jenkins/test-data/e37915a4-8313-e328-91fb-37d541e8b214/data/default/testReplayEditsAfterRegionMovedWithMultiCF/567d9cab2ff71509e601db9c0fa44478/.tmp/cf1/b885b098c54744cd9d0d8b9ad1b63b3d 2024-11-24T07:35:29,598 INFO [Time-limited test {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for b885b098c54744cd9d0d8b9ad1b63b3d 2024-11-24T07:35:29,613 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33505/user/jenkins/test-data/e37915a4-8313-e328-91fb-37d541e8b214/data/default/testReplayEditsAfterRegionMovedWithMultiCF/567d9cab2ff71509e601db9c0fa44478/.tmp/cf2/cf9b18c873dd4749a3cb98dc50d365d1 is 29, key is r1/cf2:/1732433729556/DeleteFamily/seqid=0 2024-11-24T07:35:29,620 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42685 is added to blk_1073741889_1067 (size=4906) 2024-11-24T07:35:29,620 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41161 is added to blk_1073741889_1067 (size=4906) 2024-11-24T07:35:29,620 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41971 is added to blk_1073741889_1067 (size=4906) 2024-11-24T07:35:29,621 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=25 B at sequenceid=12 (bloomFilter=false), to=hdfs://localhost:33505/user/jenkins/test-data/e37915a4-8313-e328-91fb-37d541e8b214/data/default/testReplayEditsAfterRegionMovedWithMultiCF/567d9cab2ff71509e601db9c0fa44478/.tmp/cf2/cf9b18c873dd4749a3cb98dc50d365d1 2024-11-24T07:35:29,627 INFO [Time-limited test {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for cf9b18c873dd4749a3cb98dc50d365d1 2024-11-24T07:35:29,628 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33505/user/jenkins/test-data/e37915a4-8313-e328-91fb-37d541e8b214/data/default/testReplayEditsAfterRegionMovedWithMultiCF/567d9cab2ff71509e601db9c0fa44478/.tmp/cf1/b885b098c54744cd9d0d8b9ad1b63b3d as hdfs://localhost:33505/user/jenkins/test-data/e37915a4-8313-e328-91fb-37d541e8b214/data/default/testReplayEditsAfterRegionMovedWithMultiCF/567d9cab2ff71509e601db9c0fa44478/cf1/b885b098c54744cd9d0d8b9ad1b63b3d 2024-11-24T07:35:29,634 INFO [Time-limited test {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for b885b098c54744cd9d0d8b9ad1b63b3d 2024-11-24T07:35:29,635 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33505/user/jenkins/test-data/e37915a4-8313-e328-91fb-37d541e8b214/data/default/testReplayEditsAfterRegionMovedWithMultiCF/567d9cab2ff71509e601db9c0fa44478/cf1/b885b098c54744cd9d0d8b9ad1b63b3d, entries=1, sequenceid=12, filesize=4.8 K 2024-11-24T07:35:29,636 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33505/user/jenkins/test-data/e37915a4-8313-e328-91fb-37d541e8b214/data/default/testReplayEditsAfterRegionMovedWithMultiCF/567d9cab2ff71509e601db9c0fa44478/.tmp/cf2/cf9b18c873dd4749a3cb98dc50d365d1 as hdfs://localhost:33505/user/jenkins/test-data/e37915a4-8313-e328-91fb-37d541e8b214/data/default/testReplayEditsAfterRegionMovedWithMultiCF/567d9cab2ff71509e601db9c0fa44478/cf2/cf9b18c873dd4749a3cb98dc50d365d1 2024-11-24T07:35:29,643 INFO [Time-limited test {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for cf9b18c873dd4749a3cb98dc50d365d1 2024-11-24T07:35:29,643 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33505/user/jenkins/test-data/e37915a4-8313-e328-91fb-37d541e8b214/data/default/testReplayEditsAfterRegionMovedWithMultiCF/567d9cab2ff71509e601db9c0fa44478/cf2/cf9b18c873dd4749a3cb98dc50d365d1, entries=1, sequenceid=12, filesize=4.8 K 2024-11-24T07:35:29,644 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~50 B/50, heapSize ~688 B/688, currentSize=0 B/0 for 567d9cab2ff71509e601db9c0fa44478 in 80ms, sequenceid=12, compaction requested=false 2024-11-24T07:35:29,644 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for 567d9cab2ff71509e601db9c0fa44478: 2024-11-24T07:35:29,647 DEBUG [Time-limited test {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 2 store files, 0 compacting, 2 eligible, 16 blocking 2024-11-24T07:35:29,648 DEBUG [Time-limited test {}] regionserver.HStore(1541): 567d9cab2ff71509e601db9c0fa44478/cf1 is initiating major compaction (all files) 2024-11-24T07:35:29,649 INFO [Time-limited test {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-24T07:35:29,649 INFO [Time-limited test {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-24T07:35:29,649 INFO [Time-limited test {}] regionserver.HRegion(2416): Starting compaction of 567d9cab2ff71509e601db9c0fa44478/cf1 in testReplayEditsAfterRegionMovedWithMultiCF,,1732433728060.567d9cab2ff71509e601db9c0fa44478. 2024-11-24T07:35:29,650 INFO [Time-limited test {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33505/user/jenkins/test-data/e37915a4-8313-e328-91fb-37d541e8b214/data/default/testReplayEditsAfterRegionMovedWithMultiCF/567d9cab2ff71509e601db9c0fa44478/cf1/d657c95ee1b14d3e82ed05bfbe9a2cfb, hdfs://localhost:33505/user/jenkins/test-data/e37915a4-8313-e328-91fb-37d541e8b214/data/default/testReplayEditsAfterRegionMovedWithMultiCF/567d9cab2ff71509e601db9c0fa44478/cf1/b885b098c54744cd9d0d8b9ad1b63b3d] into tmpdir=hdfs://localhost:33505/user/jenkins/test-data/e37915a4-8313-e328-91fb-37d541e8b214/data/default/testReplayEditsAfterRegionMovedWithMultiCF/567d9cab2ff71509e601db9c0fa44478/.tmp, totalSize=9.5 K 2024-11-24T07:35:29,651 DEBUG [Time-limited test {}] compactions.Compactor(225): Compacting d657c95ee1b14d3e82ed05bfbe9a2cfb, keycount=1, bloomtype=NONE, size=4.7 K, encoding=NONE, compression=NONE, seqNum=5, earliestPutTs=1732433728739 2024-11-24T07:35:29,651 DEBUG [Time-limited test {}] compactions.Compactor(225): Compacting b885b098c54744cd9d0d8b9ad1b63b3d, keycount=1, bloomtype=NONE, size=4.8 K, encoding=NONE, compression=NONE, seqNum=12, earliestPutTs=9223372036854775807 2024-11-24T07:35:29,662 INFO [Time-limited test {}] throttle.PressureAwareThroughputController(145): 567d9cab2ff71509e601db9c0fa44478#cf1#compaction#16 average throughput is NaN MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-24T07:35:29,669 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41161 is added to blk_1073741890_1068 (size=4626) 2024-11-24T07:35:29,670 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42685 is added to blk_1073741890_1068 (size=4626) 2024-11-24T07:35:29,670 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41971 is added to blk_1073741890_1068 (size=4626) 2024-11-24T07:35:29,677 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33505/user/jenkins/test-data/e37915a4-8313-e328-91fb-37d541e8b214/data/default/testReplayEditsAfterRegionMovedWithMultiCF/567d9cab2ff71509e601db9c0fa44478/.tmp/cf1/30e9069526ab4d65b8c0c6945d917fc5 as hdfs://localhost:33505/user/jenkins/test-data/e37915a4-8313-e328-91fb-37d541e8b214/data/default/testReplayEditsAfterRegionMovedWithMultiCF/567d9cab2ff71509e601db9c0fa44478/cf1/30e9069526ab4d65b8c0c6945d917fc5 2024-11-24T07:35:29,690 INFO [Time-limited test {}] regionserver.HStore(1337): Completed major compaction of 2 (all) file(s) in 567d9cab2ff71509e601db9c0fa44478/cf1 of 567d9cab2ff71509e601db9c0fa44478 into 30e9069526ab4d65b8c0c6945d917fc5(size=4.5 K), total size for store is 4.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T07:35:29,690 DEBUG [Time-limited test {}] regionserver.HRegion(2446): Compaction status journal for 567d9cab2ff71509e601db9c0fa44478: 2024-11-24T07:35:29,690 DEBUG [Time-limited test {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 1 store files, 0 compacting, 1 eligible, 16 blocking 2024-11-24T07:35:29,691 DEBUG [Time-limited test {}] regionserver.HStore(1541): 567d9cab2ff71509e601db9c0fa44478/cf2 is initiating major compaction (all files) 2024-11-24T07:35:29,691 INFO [Time-limited test {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-24T07:35:29,691 INFO [Time-limited test {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-24T07:35:29,691 INFO [Time-limited test {}] regionserver.HRegion(2416): Starting compaction of 567d9cab2ff71509e601db9c0fa44478/cf2 in testReplayEditsAfterRegionMovedWithMultiCF,,1732433728060.567d9cab2ff71509e601db9c0fa44478. 2024-11-24T07:35:29,691 INFO [Time-limited test {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33505/user/jenkins/test-data/e37915a4-8313-e328-91fb-37d541e8b214/data/default/testReplayEditsAfterRegionMovedWithMultiCF/567d9cab2ff71509e601db9c0fa44478/cf2/cf9b18c873dd4749a3cb98dc50d365d1] into tmpdir=hdfs://localhost:33505/user/jenkins/test-data/e37915a4-8313-e328-91fb-37d541e8b214/data/default/testReplayEditsAfterRegionMovedWithMultiCF/567d9cab2ff71509e601db9c0fa44478/.tmp, totalSize=4.8 K 2024-11-24T07:35:29,692 DEBUG [Time-limited test {}] compactions.Compactor(225): Compacting cf9b18c873dd4749a3cb98dc50d365d1, keycount=1, bloomtype=NONE, size=4.8 K, encoding=NONE, compression=NONE, seqNum=12, earliestPutTs=9223372036854775807 2024-11-24T07:35:29,698 INFO [Time-limited test {}] throttle.PressureAwareThroughputController(145): 567d9cab2ff71509e601db9c0fa44478#cf2#compaction#17 average throughput is NaN MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-24T07:35:29,706 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41161 is added to blk_1073741891_1069 (size=4592) 2024-11-24T07:35:29,706 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41971 is added to blk_1073741891_1069 (size=4592) 2024-11-24T07:35:29,707 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42685 is added to blk_1073741891_1069 (size=4592) 2024-11-24T07:35:29,715 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33505/user/jenkins/test-data/e37915a4-8313-e328-91fb-37d541e8b214/data/default/testReplayEditsAfterRegionMovedWithMultiCF/567d9cab2ff71509e601db9c0fa44478/.tmp/cf2/476cd101fd09468e8d2393440995e674 as hdfs://localhost:33505/user/jenkins/test-data/e37915a4-8313-e328-91fb-37d541e8b214/data/default/testReplayEditsAfterRegionMovedWithMultiCF/567d9cab2ff71509e601db9c0fa44478/cf2/476cd101fd09468e8d2393440995e674 2024-11-24T07:35:29,723 INFO [Time-limited test {}] regionserver.HStore(1337): Completed major compaction of 1 (all) file(s) in 567d9cab2ff71509e601db9c0fa44478/cf2 of 567d9cab2ff71509e601db9c0fa44478 into 476cd101fd09468e8d2393440995e674(size=4.5 K), total size for store is 4.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T07:35:29,723 DEBUG [Time-limited test {}] regionserver.HRegion(2446): Compaction status journal for 567d9cab2ff71509e601db9c0fa44478: 2024-11-24T07:35:29,728 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37719 {}] master.HMaster(2410): Client=jenkins//172.17.0.3 move hri=567d9cab2ff71509e601db9c0fa44478, source=1ab588ae992e,46433,1732433713473, destination=1ab588ae992e,40611,1732433713633, warming up region on 1ab588ae992e,40611,1732433713633 2024-11-24T07:35:29,728 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37719 {}] master.HMaster(2414): Client=jenkins//172.17.0.3 move hri=567d9cab2ff71509e601db9c0fa44478, source=1ab588ae992e,46433,1732433713473, destination=1ab588ae992e,40611,1732433713633, running balancer 2024-11-24T07:35:29,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37719 {}] procedure2.ProcedureExecutor(1139): Stored pid=10, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=567d9cab2ff71509e601db9c0fa44478, REOPEN/MOVE 2024-11-24T07:35:29,730 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=10, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=567d9cab2ff71509e601db9c0fa44478, REOPEN/MOVE 2024-11-24T07:35:29,731 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=10 updating hbase:meta row=567d9cab2ff71509e601db9c0fa44478, regionState=CLOSING, regionLocation=1ab588ae992e,46433,1732433713473 2024-11-24T07:35:29,732 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40611 {}] regionserver.RSRpcServices(2066): Warmup testReplayEditsAfterRegionMovedWithMultiCF,,1732433728060.567d9cab2ff71509e601db9c0fa44478. 2024-11-24T07:35:29,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40611 {}] regionserver.HRegion(7855): Warmup {ENCODED => 567d9cab2ff71509e601db9c0fa44478, NAME => 'testReplayEditsAfterRegionMovedWithMultiCF,,1732433728060.567d9cab2ff71509e601db9c0fa44478.', STARTKEY => '', ENDKEY => ''} 2024-11-24T07:35:29,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40611 {}] regionserver.HRegion(898): Instantiated testReplayEditsAfterRegionMovedWithMultiCF,,1732433728060.567d9cab2ff71509e601db9c0fa44478.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T07:35:29,733 INFO [StoreOpener-567d9cab2ff71509e601db9c0fa44478-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf1 of region 567d9cab2ff71509e601db9c0fa44478 2024-11-24T07:35:29,734 INFO [StoreOpener-567d9cab2ff71509e601db9c0fa44478-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 567d9cab2ff71509e601db9c0fa44478 columnFamilyName cf1 2024-11-24T07:35:29,734 DEBUG [StoreOpener-567d9cab2ff71509e601db9c0fa44478-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T07:35:29,734 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=10, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=567d9cab2ff71509e601db9c0fa44478, REOPEN/MOVE because future has completed 2024-11-24T07:35:29,735 DEBUG [PEWorker-5 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-24T07:35:29,735 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=11, ppid=10, state=RUNNABLE, hasLock=false; CloseRegionProcedure 567d9cab2ff71509e601db9c0fa44478, server=1ab588ae992e,46433,1732433713473}] 2024-11-24T07:35:29,746 DEBUG [StoreOpener-567d9cab2ff71509e601db9c0fa44478-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:33505/user/jenkins/test-data/e37915a4-8313-e328-91fb-37d541e8b214/data/default/testReplayEditsAfterRegionMovedWithMultiCF/567d9cab2ff71509e601db9c0fa44478/cf1/30e9069526ab4d65b8c0c6945d917fc5 2024-11-24T07:35:29,752 INFO [StoreFileOpener-567d9cab2ff71509e601db9c0fa44478-cf1-1 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for b885b098c54744cd9d0d8b9ad1b63b3d 2024-11-24T07:35:29,752 DEBUG [StoreOpener-567d9cab2ff71509e601db9c0fa44478-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:33505/user/jenkins/test-data/e37915a4-8313-e328-91fb-37d541e8b214/data/default/testReplayEditsAfterRegionMovedWithMultiCF/567d9cab2ff71509e601db9c0fa44478/cf1/b885b098c54744cd9d0d8b9ad1b63b3d 2024-11-24T07:35:29,758 DEBUG [StoreOpener-567d9cab2ff71509e601db9c0fa44478-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:33505/user/jenkins/test-data/e37915a4-8313-e328-91fb-37d541e8b214/data/default/testReplayEditsAfterRegionMovedWithMultiCF/567d9cab2ff71509e601db9c0fa44478/cf1/d657c95ee1b14d3e82ed05bfbe9a2cfb 2024-11-24T07:35:29,759 INFO [StoreOpener-567d9cab2ff71509e601db9c0fa44478-1 {}] regionserver.HStore(327): Store=567d9cab2ff71509e601db9c0fa44478/cf1, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T07:35:29,759 INFO [StoreOpener-567d9cab2ff71509e601db9c0fa44478-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf2 of region 567d9cab2ff71509e601db9c0fa44478 2024-11-24T07:35:29,760 INFO [StoreOpener-567d9cab2ff71509e601db9c0fa44478-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 567d9cab2ff71509e601db9c0fa44478 columnFamilyName cf2 2024-11-24T07:35:29,760 DEBUG [StoreOpener-567d9cab2ff71509e601db9c0fa44478-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T07:35:29,767 DEBUG [StoreOpener-567d9cab2ff71509e601db9c0fa44478-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:33505/user/jenkins/test-data/e37915a4-8313-e328-91fb-37d541e8b214/data/default/testReplayEditsAfterRegionMovedWithMultiCF/567d9cab2ff71509e601db9c0fa44478/cf2/476cd101fd09468e8d2393440995e674 2024-11-24T07:35:29,773 INFO [StoreFileOpener-567d9cab2ff71509e601db9c0fa44478-cf2-1 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for cf9b18c873dd4749a3cb98dc50d365d1 2024-11-24T07:35:29,773 DEBUG [StoreOpener-567d9cab2ff71509e601db9c0fa44478-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:33505/user/jenkins/test-data/e37915a4-8313-e328-91fb-37d541e8b214/data/default/testReplayEditsAfterRegionMovedWithMultiCF/567d9cab2ff71509e601db9c0fa44478/cf2/cf9b18c873dd4749a3cb98dc50d365d1 2024-11-24T07:35:29,773 INFO [StoreOpener-567d9cab2ff71509e601db9c0fa44478-1 {}] regionserver.HStore(327): Store=567d9cab2ff71509e601db9c0fa44478/cf2, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T07:35:29,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40611 {}] regionserver.HRegion(1722): Closing 567d9cab2ff71509e601db9c0fa44478, disabling compactions & flushes 2024-11-24T07:35:29,774 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40611 {}] regionserver.HRegion(1755): Closing region testReplayEditsAfterRegionMovedWithMultiCF,,1732433728060.567d9cab2ff71509e601db9c0fa44478. 2024-11-24T07:35:29,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40611 {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1732433728060.567d9cab2ff71509e601db9c0fa44478. 2024-11-24T07:35:29,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40611 {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1732433728060.567d9cab2ff71509e601db9c0fa44478. after waiting 0 ms 2024-11-24T07:35:29,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40611 {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsAfterRegionMovedWithMultiCF,,1732433728060.567d9cab2ff71509e601db9c0fa44478. 2024-11-24T07:35:29,775 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40611 {}] regionserver.HRegion(1973): Closed testReplayEditsAfterRegionMovedWithMultiCF,,1732433728060.567d9cab2ff71509e601db9c0fa44478. 2024-11-24T07:35:29,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40611 {}] regionserver.HRegion(1676): Region close journal for 567d9cab2ff71509e601db9c0fa44478: Waiting for close lock at 1732433729773Disabling compacts and flushes for region at 1732433729774 (+1 ms)Disabling writes for close at 1732433729774Writing region close event to WAL at 1732433729775 (+1 ms)Closed at 1732433729775 2024-11-24T07:35:29,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37719 {}] procedure.ProcedureSyncWait(219): waitFor pid=10 2024-11-24T07:35:29,891 INFO [RS_CLOSE_REGION-regionserver/1ab588ae992e:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] handler.UnassignRegionHandler(122): Close 567d9cab2ff71509e601db9c0fa44478 2024-11-24T07:35:29,891 DEBUG [RS_CLOSE_REGION-regionserver/1ab588ae992e:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-24T07:35:29,891 DEBUG [RS_CLOSE_REGION-regionserver/1ab588ae992e:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] regionserver.HRegion(1722): Closing 567d9cab2ff71509e601db9c0fa44478, disabling compactions & flushes 2024-11-24T07:35:29,891 INFO [RS_CLOSE_REGION-regionserver/1ab588ae992e:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] regionserver.HRegion(1755): Closing region testReplayEditsAfterRegionMovedWithMultiCF,,1732433728060.567d9cab2ff71509e601db9c0fa44478. 2024-11-24T07:35:29,891 DEBUG [RS_CLOSE_REGION-regionserver/1ab588ae992e:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1732433728060.567d9cab2ff71509e601db9c0fa44478. 2024-11-24T07:35:29,891 DEBUG [RS_CLOSE_REGION-regionserver/1ab588ae992e:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1732433728060.567d9cab2ff71509e601db9c0fa44478. after waiting 0 ms 2024-11-24T07:35:29,891 DEBUG [RS_CLOSE_REGION-regionserver/1ab588ae992e:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsAfterRegionMovedWithMultiCF,,1732433728060.567d9cab2ff71509e601db9c0fa44478. 2024-11-24T07:35:29,892 DEBUG [StoreCloser-testReplayEditsAfterRegionMovedWithMultiCF,,1732433728060.567d9cab2ff71509e601db9c0fa44478.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:33505/user/jenkins/test-data/e37915a4-8313-e328-91fb-37d541e8b214/data/default/testReplayEditsAfterRegionMovedWithMultiCF/567d9cab2ff71509e601db9c0fa44478/cf1/d657c95ee1b14d3e82ed05bfbe9a2cfb, hdfs://localhost:33505/user/jenkins/test-data/e37915a4-8313-e328-91fb-37d541e8b214/data/default/testReplayEditsAfterRegionMovedWithMultiCF/567d9cab2ff71509e601db9c0fa44478/cf1/b885b098c54744cd9d0d8b9ad1b63b3d] to archive 2024-11-24T07:35:29,894 DEBUG [StoreCloser-testReplayEditsAfterRegionMovedWithMultiCF,,1732433728060.567d9cab2ff71509e601db9c0fa44478.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-24T07:35:29,898 DEBUG [StoreCloser-testReplayEditsAfterRegionMovedWithMultiCF,,1732433728060.567d9cab2ff71509e601db9c0fa44478.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33505/user/jenkins/test-data/e37915a4-8313-e328-91fb-37d541e8b214/data/default/testReplayEditsAfterRegionMovedWithMultiCF/567d9cab2ff71509e601db9c0fa44478/cf1/d657c95ee1b14d3e82ed05bfbe9a2cfb to hdfs://localhost:33505/user/jenkins/test-data/e37915a4-8313-e328-91fb-37d541e8b214/archive/data/default/testReplayEditsAfterRegionMovedWithMultiCF/567d9cab2ff71509e601db9c0fa44478/cf1/d657c95ee1b14d3e82ed05bfbe9a2cfb 2024-11-24T07:35:29,899 DEBUG [StoreCloser-testReplayEditsAfterRegionMovedWithMultiCF,,1732433728060.567d9cab2ff71509e601db9c0fa44478.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33505/user/jenkins/test-data/e37915a4-8313-e328-91fb-37d541e8b214/data/default/testReplayEditsAfterRegionMovedWithMultiCF/567d9cab2ff71509e601db9c0fa44478/cf1/b885b098c54744cd9d0d8b9ad1b63b3d to hdfs://localhost:33505/user/jenkins/test-data/e37915a4-8313-e328-91fb-37d541e8b214/archive/data/default/testReplayEditsAfterRegionMovedWithMultiCF/567d9cab2ff71509e601db9c0fa44478/cf1/b885b098c54744cd9d0d8b9ad1b63b3d 2024-11-24T07:35:29,911 DEBUG [StoreCloser-testReplayEditsAfterRegionMovedWithMultiCF,,1732433728060.567d9cab2ff71509e601db9c0fa44478.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:33505/user/jenkins/test-data/e37915a4-8313-e328-91fb-37d541e8b214/data/default/testReplayEditsAfterRegionMovedWithMultiCF/567d9cab2ff71509e601db9c0fa44478/cf2/cf9b18c873dd4749a3cb98dc50d365d1] to archive 2024-11-24T07:35:29,913 DEBUG [StoreCloser-testReplayEditsAfterRegionMovedWithMultiCF,,1732433728060.567d9cab2ff71509e601db9c0fa44478.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-24T07:35:29,915 DEBUG [StoreCloser-testReplayEditsAfterRegionMovedWithMultiCF,,1732433728060.567d9cab2ff71509e601db9c0fa44478.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33505/user/jenkins/test-data/e37915a4-8313-e328-91fb-37d541e8b214/data/default/testReplayEditsAfterRegionMovedWithMultiCF/567d9cab2ff71509e601db9c0fa44478/cf2/cf9b18c873dd4749a3cb98dc50d365d1 to hdfs://localhost:33505/user/jenkins/test-data/e37915a4-8313-e328-91fb-37d541e8b214/archive/data/default/testReplayEditsAfterRegionMovedWithMultiCF/567d9cab2ff71509e601db9c0fa44478/cf2/cf9b18c873dd4749a3cb98dc50d365d1 2024-11-24T07:35:29,921 DEBUG [RS_CLOSE_REGION-regionserver/1ab588ae992e:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:33505/user/jenkins/test-data/e37915a4-8313-e328-91fb-37d541e8b214/data/default/testReplayEditsAfterRegionMovedWithMultiCF/567d9cab2ff71509e601db9c0fa44478/recovered.edits/17.seqid, newMaxSeqId=17, maxSeqId=8 2024-11-24T07:35:29,922 INFO [RS_CLOSE_REGION-regionserver/1ab588ae992e:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] regionserver.HRegion(1973): Closed testReplayEditsAfterRegionMovedWithMultiCF,,1732433728060.567d9cab2ff71509e601db9c0fa44478. 2024-11-24T07:35:29,922 DEBUG [RS_CLOSE_REGION-regionserver/1ab588ae992e:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] regionserver.HRegion(1676): Region close journal for 567d9cab2ff71509e601db9c0fa44478: Waiting for close lock at 1732433729891Running coprocessor pre-close hooks at 1732433729891Disabling compacts and flushes for region at 1732433729891Disabling writes for close at 1732433729891Writing region close event to WAL at 1732433729916 (+25 ms)Running coprocessor post-close hooks at 1732433729922 (+6 ms)Closed at 1732433729922 2024-11-24T07:35:29,922 INFO [RS_CLOSE_REGION-regionserver/1ab588ae992e:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] regionserver.HRegionServer(3302): Adding 567d9cab2ff71509e601db9c0fa44478 move to 1ab588ae992e,40611,1732433713633 record at close sequenceid=12 2024-11-24T07:35:29,925 INFO [RS_CLOSE_REGION-regionserver/1ab588ae992e:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] handler.UnassignRegionHandler(157): Closed 567d9cab2ff71509e601db9c0fa44478 2024-11-24T07:35:29,926 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=10 updating hbase:meta row=567d9cab2ff71509e601db9c0fa44478, regionState=CLOSED 2024-11-24T07:35:29,929 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=11, ppid=10, state=RUNNABLE, hasLock=false; CloseRegionProcedure 567d9cab2ff71509e601db9c0fa44478, server=1ab588ae992e,46433,1732433713473 because future has completed 2024-11-24T07:35:29,933 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=11, resume processing ppid=10 2024-11-24T07:35:29,933 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=11, ppid=10, state=SUCCESS, hasLock=false; CloseRegionProcedure 567d9cab2ff71509e601db9c0fa44478, server=1ab588ae992e,46433,1732433713473 in 195 msec 2024-11-24T07:35:29,934 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=10, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=567d9cab2ff71509e601db9c0fa44478, REOPEN/MOVE; state=CLOSED, location=1ab588ae992e,40611,1732433713633; forceNewPlan=false, retain=false 2024-11-24T07:35:30,085 INFO [1ab588ae992e:37719 {}] balancer.BaseLoadBalancer(388): Reassigned 1 regions. 1 retained the pre-restart assignment. 2024-11-24T07:35:30,086 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=10 updating hbase:meta row=567d9cab2ff71509e601db9c0fa44478, regionState=OPENING, regionLocation=1ab588ae992e,40611,1732433713633 2024-11-24T07:35:30,092 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=10, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=567d9cab2ff71509e601db9c0fa44478, REOPEN/MOVE because future has completed 2024-11-24T07:35:30,092 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=12, ppid=10, state=RUNNABLE, hasLock=false; OpenRegionProcedure 567d9cab2ff71509e601db9c0fa44478, server=1ab588ae992e,40611,1732433713633}] 2024-11-24T07:35:30,250 INFO [RS_OPEN_REGION-regionserver/1ab588ae992e:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] handler.AssignRegionHandler(132): Open testReplayEditsAfterRegionMovedWithMultiCF,,1732433728060.567d9cab2ff71509e601db9c0fa44478. 2024-11-24T07:35:30,250 DEBUG [RS_OPEN_REGION-regionserver/1ab588ae992e:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(7752): Opening region: {ENCODED => 567d9cab2ff71509e601db9c0fa44478, NAME => 'testReplayEditsAfterRegionMovedWithMultiCF,,1732433728060.567d9cab2ff71509e601db9c0fa44478.', STARTKEY => '', ENDKEY => ''} 2024-11-24T07:35:30,250 DEBUG [RS_OPEN_REGION-regionserver/1ab588ae992e:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testReplayEditsAfterRegionMovedWithMultiCF 567d9cab2ff71509e601db9c0fa44478 2024-11-24T07:35:30,251 DEBUG [RS_OPEN_REGION-regionserver/1ab588ae992e:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(898): Instantiated testReplayEditsAfterRegionMovedWithMultiCF,,1732433728060.567d9cab2ff71509e601db9c0fa44478.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T07:35:30,251 DEBUG [RS_OPEN_REGION-regionserver/1ab588ae992e:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(7794): checking encryption for 567d9cab2ff71509e601db9c0fa44478 2024-11-24T07:35:30,251 DEBUG [RS_OPEN_REGION-regionserver/1ab588ae992e:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(7797): checking classloading for 567d9cab2ff71509e601db9c0fa44478 2024-11-24T07:35:30,253 INFO [StoreOpener-567d9cab2ff71509e601db9c0fa44478-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf1 of region 567d9cab2ff71509e601db9c0fa44478 2024-11-24T07:35:30,255 INFO [StoreOpener-567d9cab2ff71509e601db9c0fa44478-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 567d9cab2ff71509e601db9c0fa44478 columnFamilyName cf1 2024-11-24T07:35:30,255 DEBUG [StoreOpener-567d9cab2ff71509e601db9c0fa44478-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T07:35:30,265 DEBUG [StoreOpener-567d9cab2ff71509e601db9c0fa44478-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:33505/user/jenkins/test-data/e37915a4-8313-e328-91fb-37d541e8b214/data/default/testReplayEditsAfterRegionMovedWithMultiCF/567d9cab2ff71509e601db9c0fa44478/cf1/30e9069526ab4d65b8c0c6945d917fc5 2024-11-24T07:35:30,265 INFO [StoreOpener-567d9cab2ff71509e601db9c0fa44478-1 {}] regionserver.HStore(327): Store=567d9cab2ff71509e601db9c0fa44478/cf1, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T07:35:30,265 INFO [StoreOpener-567d9cab2ff71509e601db9c0fa44478-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf2 of region 567d9cab2ff71509e601db9c0fa44478 2024-11-24T07:35:30,267 INFO [StoreOpener-567d9cab2ff71509e601db9c0fa44478-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 567d9cab2ff71509e601db9c0fa44478 columnFamilyName cf2 2024-11-24T07:35:30,267 DEBUG [StoreOpener-567d9cab2ff71509e601db9c0fa44478-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T07:35:30,273 DEBUG [StoreOpener-567d9cab2ff71509e601db9c0fa44478-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:33505/user/jenkins/test-data/e37915a4-8313-e328-91fb-37d541e8b214/data/default/testReplayEditsAfterRegionMovedWithMultiCF/567d9cab2ff71509e601db9c0fa44478/cf2/476cd101fd09468e8d2393440995e674 2024-11-24T07:35:30,273 INFO [StoreOpener-567d9cab2ff71509e601db9c0fa44478-1 {}] regionserver.HStore(327): Store=567d9cab2ff71509e601db9c0fa44478/cf2, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T07:35:30,273 DEBUG [RS_OPEN_REGION-regionserver/1ab588ae992e:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1038): replaying wal for 567d9cab2ff71509e601db9c0fa44478 2024-11-24T07:35:30,274 DEBUG [RS_OPEN_REGION-regionserver/1ab588ae992e:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33505/user/jenkins/test-data/e37915a4-8313-e328-91fb-37d541e8b214/data/default/testReplayEditsAfterRegionMovedWithMultiCF/567d9cab2ff71509e601db9c0fa44478 2024-11-24T07:35:30,276 DEBUG [RS_OPEN_REGION-regionserver/1ab588ae992e:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33505/user/jenkins/test-data/e37915a4-8313-e328-91fb-37d541e8b214/data/default/testReplayEditsAfterRegionMovedWithMultiCF/567d9cab2ff71509e601db9c0fa44478 2024-11-24T07:35:30,277 DEBUG [RS_OPEN_REGION-regionserver/1ab588ae992e:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1048): stopping wal replay for 567d9cab2ff71509e601db9c0fa44478 2024-11-24T07:35:30,277 DEBUG [RS_OPEN_REGION-regionserver/1ab588ae992e:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1060): Cleaning up temporary data for 567d9cab2ff71509e601db9c0fa44478 2024-11-24T07:35:30,278 DEBUG [RS_OPEN_REGION-regionserver/1ab588ae992e:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsAfterRegionMovedWithMultiCF descriptor;using region.getMemStoreFlushHeapSize/# of families (64.0 M)) instead. 2024-11-24T07:35:30,279 DEBUG [RS_OPEN_REGION-regionserver/1ab588ae992e:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1093): writing seq id for 567d9cab2ff71509e601db9c0fa44478 2024-11-24T07:35:30,280 INFO [RS_OPEN_REGION-regionserver/1ab588ae992e:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1114): Opened 567d9cab2ff71509e601db9c0fa44478; next sequenceid=18; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=62118978, jitterRate=-0.0743550956249237}}}, FlushLargeStoresPolicy{flushSizeLowerBound=67108864} 2024-11-24T07:35:30,281 DEBUG [RS_OPEN_REGION-regionserver/1ab588ae992e:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 567d9cab2ff71509e601db9c0fa44478 2024-11-24T07:35:30,282 DEBUG [RS_OPEN_REGION-regionserver/1ab588ae992e:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1006): Region open journal for 567d9cab2ff71509e601db9c0fa44478: Running coprocessor pre-open hook at 1732433730251Writing region info on filesystem at 1732433730251Initializing all the Stores at 1732433730253 (+2 ms)Instantiating store for column family {NAME => 'cf1', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732433730253Instantiating store for column family {NAME => 'cf2', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732433730253Cleaning up temporary data from old regions at 1732433730277 (+24 ms)Running coprocessor post-open hooks at 1732433730281 (+4 ms)Region opened successfully at 1732433730281 2024-11-24T07:35:30,283 INFO [RS_OPEN_REGION-regionserver/1ab588ae992e:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegionServer(2236): Post open deploy tasks for testReplayEditsAfterRegionMovedWithMultiCF,,1732433728060.567d9cab2ff71509e601db9c0fa44478., pid=12, masterSystemTime=1732433730245 2024-11-24T07:35:30,285 DEBUG [RS_OPEN_REGION-regionserver/1ab588ae992e:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegionServer(2266): Finished post open deploy task for testReplayEditsAfterRegionMovedWithMultiCF,,1732433728060.567d9cab2ff71509e601db9c0fa44478. 2024-11-24T07:35:30,285 INFO [RS_OPEN_REGION-regionserver/1ab588ae992e:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] handler.AssignRegionHandler(153): Opened testReplayEditsAfterRegionMovedWithMultiCF,,1732433728060.567d9cab2ff71509e601db9c0fa44478. 2024-11-24T07:35:30,286 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=10 updating hbase:meta row=567d9cab2ff71509e601db9c0fa44478, regionState=OPEN, openSeqNum=18, regionLocation=1ab588ae992e,40611,1732433713633 2024-11-24T07:35:30,288 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=12, ppid=10, state=RUNNABLE, hasLock=false; OpenRegionProcedure 567d9cab2ff71509e601db9c0fa44478, server=1ab588ae992e,40611,1732433713633 because future has completed 2024-11-24T07:35:30,292 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=12, resume processing ppid=10 2024-11-24T07:35:30,292 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=12, ppid=10, state=SUCCESS, hasLock=false; OpenRegionProcedure 567d9cab2ff71509e601db9c0fa44478, server=1ab588ae992e,40611,1732433713633 in 198 msec 2024-11-24T07:35:30,294 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=10, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=567d9cab2ff71509e601db9c0fa44478, REOPEN/MOVE in 564 msec 2024-11-24T07:35:30,332 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-24T07:35:30,335 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:41640, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-24T07:35:30,338 ERROR [Time-limited test {}] regionserver.HRegionServer(2442): ***** ABORTING region server 1ab588ae992e,40611,1732433713633: testing ***** 2024-11-24T07:35:30,338 ERROR [Time-limited test {}] regionserver.HRegionServer(2447): RegionServer abort: loaded coprocessors are: [org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint] 2024-11-24T07:35:30,343 DEBUG [Time-limited test {}] util.JSONBean(135): Listing beans for java.lang:type=Memory 2024-11-24T07:35:30,344 DEBUG [Time-limited test {}] util.JSONBean(135): Listing beans for Hadoop:service=HBase,name=RegionServer,sub=IPC 2024-11-24T07:35:30,348 DEBUG [Time-limited test {}] util.JSONBean(135): Listing beans for Hadoop:service=HBase,name=RegionServer,sub=Replication 2024-11-24T07:35:30,350 DEBUG [Time-limited test {}] util.JSONBean(135): Listing beans for Hadoop:service=HBase,name=RegionServer,sub=Server 2024-11-24T07:35:30,360 INFO [Time-limited test {}] regionserver.HRegionServer(2451): Dump of metrics as JSON on abort: { "beans": [ { "name": "java.lang:type=Memory", "modelerType": "sun.management.MemoryImpl", "ObjectPendingFinalizationCount": 0, "HeapMemoryUsage": { "committed": 1048576000, "init": 1048576000, "max": 2306867200, "used": 507901336 }, "NonHeapMemoryUsage": { "committed": 171180032, "init": 7667712, "max": -1, "used": 168646184 }, "Verbose": false, "ObjectName": "java.lang:type=Memory" } ], "beans": [ { "name": "Hadoop:service=HBase,name=RegionServer,sub=IPC", "modelerType": "RegionServer,sub=IPC", "tag.Context": "regionserver", "tag.Hostname": "1ab588ae992e", "queueSize": 0, "numCallsInGeneralQueue": 0, "numCallsInReplicationQueue": 0, "numCallsInBulkLoadQueue": 0, "numCallsInPriorityQueue": 0, "numCallsInMetaPriorityQueue": 0, "numOpenConnections": 0, "numActiveHandler": 0, "numActiveGeneralHandler": 0, "numActivePriorityHandler": 0, "numActiveReplicationHandler": 0, "numGeneralCallsDropped": 0, "numLifoModeSwitches": 0, "numCallsInWriteQueue": 0, "numActiveBulkLoadHandler": 0, "numCallsInReadQueue": 0, "numCallsInScanQueue": 0, "numActiveWriteHandler": 0, "numActiveReadHandler": 0, "numActiveScanHandler": 0, "nettyDirectMemoryUsage": 67108864, "nettyTotalPendingOutboundBytes": 0, "nettyMaxPendingOutboundBytes": 0, "receivedBytes": 2071, "exceptions.RegionMovedException": 0, "authenticationSuccesses": 0, "authorizationFailures": 0, "exceptions.requestTooBig": 0, "UnwritableTime_num_ops": 0, "UnwritableTime_min": 0, "UnwritableTime_max": 0, "UnwritableTime_mean": 0, "UnwritableTime_25th_percentile": 0, "UnwritableTime_median": 0, "UnwritableTime_75th_percentile": 0, "UnwritableTime_90th_percentile": 0, "UnwritableTime_95th_percentile": 0, "UnwritableTime_98th_percentile": 0, "UnwritableTime_99th_percentile": 0, "UnwritableTime_99.9th_percentile": 0, "exceptions.OutOfOrderScannerNextException": 0, "exceptions.rpcThrottling": 0, "exceptions.otherExceptions": 0, "ProcessCallTime_num_ops": 8, "ProcessCallTime_min": 0, "ProcessCallTime_max": 12, "ProcessCallTime_mean": 4, "ProcessCallTime_25th_percentile": 3, "ProcessCallTime_median": 6, "ProcessCallTime_75th_percentile": 9, "ProcessCallTime_90th_percentile": 10, "ProcessCallTime_95th_percentile": 11, "ProcessCallTime_98th_percentile": 11, "ProcessCallTime_99th_percentile": 11, "ProcessCallTime_99.9th_percentile": 11, "ProcessCallTime_TimeRangeCount_0-1": 8, "exceptions.callQueueTooBig": 0, "QueueCallTime_num_ops": 8, "QueueCallTime_min": 0, "QueueCallTime_max": 0, "QueueCallTime_mean": 0, "QueueCallTime_25th_percentile": 0, "QueueCallTime_median": 0, "QueueCallTime_75th_percentile": 0, "QueueCallTime_90th_percentile": 0, "QueueCallTime_95th_percentile": 0, "QueueCallTime_98th_percentile": 0, "QueueCallTime_99th_percentile": 0, "QueueCallTime_99.9th_percentile": 0, "QueueCallTime_TimeRangeCount_0-1": 8, "authenticationFailures": 0, "exceptions.multiResponseTooLarge": 0, "exceptions.callDropped": 0, "TotalCallTime_num_ops": 8, "TotalCallTime_min": 0, "TotalCallTime_max": 12, "TotalCallTime_mean": 4, "TotalCallTime_25th_percentile": 3, "TotalCallTime_median": 6, "TotalCallTime_75th_percentile": 9, "TotalCallTime_90th_percentile": 10, "TotalCallTime_95th_percentile": 11, "TotalCallTime_98th_percentile": 11, "TotalCallTime_99th_percentile": 11, "TotalCallTime_99.9th_percentile": 11, "TotalCallTime_TimeRangeCount_0-1": 8, "exceptions.RegionTooBusyException": 0, "exceptions.FailedSanityCheckException": 0, "ResponseSize_num_ops": 8, "ResponseSize_min": 0, "ResponseSize_max": 175, "ResponseSize_mean": 50, "ResponseSize_25th_percentile": 43, "ResponseSize_median": 87, "ResponseSize_75th_percentile": 131, "ResponseSize_90th_percentile": 157, "ResponseSize_95th_percentile": 166, "ResponseSize_98th_percentile": 171, "ResponseSize_99th_percentile": 173, "ResponseSize_99.9th_percentile": 174, "ResponseSize_SizeRangeCount_0-10": 8, "exceptions.UnknownScannerException": 0, "exceptions": 0, "maxOutboundBytesExceeded": 0, "authenticationFallbacks": 0, "exceptions.quotaExceeded": 0, "exceptions.callTimedOut": 0, "exceptions.NotServingRegionException": 0, "authorizationSuccesses": 0, "exceptions.ScannerResetException": 0, "RequestSize_num_ops": 8, "RequestSize_min": 89, "RequestSize_max": 121, "RequestSize_mean": 103, "RequestSize_25th_percentile": 97, "RequestSize_median": 105, "RequestSize_75th_percentile": 113, "RequestSize_90th_percentile": 117, "RequestSize_95th_percentile": 119, "RequestSize_98th_percentile": 120, "RequestSize_99th_percentile": 120, "RequestSize_99.9th_percentile": 120, "RequestSize_SizeRangeCount_0-10": 8, "sentBytes": 352 } ], "beans": [ { "name": "Hadoop:service=HBase,name=RegionServer,sub=Replication", "modelerType": "RegionServer,sub=Replication", "tag.Context": "regionserver", "tag.Hostname": "1ab588ae992e", "source.shippedHFiles": 0, "Source.ageOfLastShippedOp_num_ops": 0, "Source.ageOfLastShippedOp_min": 0, "Source.ageOfLastShippedOp_max": 0, "Source.ageOfLastShippedOp_mean": 0, "Source.ageOfLastShippedOp_25th_percentile": 0, "Source.ageOfLastShippedOp_median": 0, "Source.ageOfLastShippedOp_75th_percentile": 0, "Source.ageOfLastShippedOp_90th_percentile": 0, "Source.ageOfLastShippedOp_95th_percentile": 0, "Source.ageOfLastShippedOp_98th_percentile": 0, "Source.ageOfLastShippedOp_99th_percentile": 0, "Source.ageOfLastShippedOp_99.9th_percentile": 0, "source.uncleanlyClosedLogs": 0, "source.closedLogsWithUnknownFileLength": 0, "source.walReaderEditsBufferUsage": 0, "source.repeatedLogFileBytes": 0, "source.sizeOfHFileRefsQueue": 0, "source.logReadInBytes": 0, "source.completedRecoverQueues": 0, "source.sizeOfLogQueue": 0, "source.restartedLogReading": 0, "source.failedRecoverQueues": 0, "source.ignoredUncleanlyClosedLogContentsInBytes": 0, "Sink.ageOfLastAppliedOp_num_ops": 0, "Sink.ageOfLastAppliedOp_min": 0, "Sink.ageOfLastAppliedOp_max": 0, "Sink.ageOfLastAppliedOp_mean": 0, "Sink.ageOfLastAppliedOp_25th_percentile": 0, "Sink.ageOfLastAppliedOp_median": 0, "Sink.ageOfLastAppliedOp_75th_percentile": 0, "Sink.ageOfLastAppliedOp_90th_percentile": 0, "Sink.ageOfLastAppliedOp_95th_percentile": 0, "Sink.ageOfLastAppliedOp_98th_percentile": 0, "Sink.ageOfLastAppliedOp_99th_percentile": 0, "Sink.ageOfLastAppliedOp_99.9th_percentile": 0, "source.logEditsRead": 0, "source.numInitializing": 0, "source.shippedOps": 0, "sink.appliedHFiles": 0, "source.logEditsFiltered": 0, "source.shippedBytes": 0, "sink.appliedOps": 0, "source.completedLogs": 0, "source.failedBatches": 0, "sink.failedBatches": 0, "source.shippedBatches": 0, "sink.appliedBatches": 0 } ], "beans": [ { "name": "Hadoop:service=HBase,name=RegionServer,sub=Server", "modelerType": "RegionServer,sub=Server", "tag.zookeeperQuorum": "127.0.0.1:49244", "tag.serverName": "1ab588ae992e,46433,1732433713473", "tag.clusterId": "c8d56efe-e808-4116-9fd2-fabfbc66fe32", "tag.Context": "regionserver", "tag.Hostname": "1ab588ae992e", "regionCount": 0, "storeCount": 0, "hlogFileCount": 1, "hlogFileSize": 0, "storeFileCount": 0, "maxStoreFileCount": 0, "memStoreSize": 0, "memStoreHeapSize": 0, "memStoreOffHeapSize": 0, "storeFileSize": 0, "storeFileSizeGrowthRate": 0.0, "maxStoreFileAge": 0, "minStoreFileAge": 0, "avgStoreFileAge": 0, "numReferenceFiles": 0, "regionServerStartTime": 1732433713473, "averageRegionSize": 0, "storeFileIndexSize": 0, "staticIndexSize": 0, "staticBloomSize": 0, "bloomFilterRequestsCount": 0, "bloomFilterNegativeResultsCount": 0, "bloomFilterEligibleRequestsCount": 0, "mutationsWithoutWALCount": 0, "mutationsWithoutWALSize": 0, "percentFilesLocal": 0.0, "percentFilesLocalSecondaryRegions": 0.0, "totalBytesRead": 10034, "localBytesRead": 10034, "shortCircuitBytesRead": 0, "zeroCopyBytesRead": 0, "splitQueueLength": 0, "compactionQueueLength": 0, "smallCompactionQueueLength": 0, "largeCompactionQueueLength": 0, "flushQueueLength": 0, "blockCacheFreeSize": 922070024, "blockCacheCount": 0, "blockCacheDataBlockCount": 0, "blockCacheSize": 676856, "blockCacheCountHitPercent": 0.0, "blockCacheExpressHitPercent": 0.0, "l1CacheSize": 676856, "l1CacheFreeSize": 922070024, "l1CacheCount": 0, "l1CacheEvictionCount": 0, "l1CacheHitCount": 0, "l1CacheMissCount": 0, "l1CacheHitRatio": 0.0, "l1CacheMissRatio": 0.0, "l2CacheSize": 0, "l2CacheFreeSize": 0, "l2CacheCount": 0, "l2CacheEvictionCount": 0, "l2CacheHitCount": 0, "l2CacheMissCount": 0, "l2CacheHitRatio": 0.0, "l2CacheMissRatio": 0.0, "mobFileCacheCount": 0, "mobFileCacheHitPercent": 0.0, "readRequestRatePerSecond": 0.0, "writeRequestRatePerSecond": 0.0, "ByteBuffAllocatorHeapAllocationBytes": 4782, "ByteBuffAllocatorPoolAllocationBytes": 0, "ByteBuffAllocatorHeapAllocationRatio": 0.0, "ByteBuffAllocatorTotalBufferCount": 186, "ByteBuffAllocatorUsedBufferCount": 0, "activeScanners": 0, "totalRequestCount": 0, "totalRowActionRequestCount": 0, "readRequestCount": 0, "cpRequestCount": 0, "filteredReadRequestCount": 0, "writeRequestCount": 0, "rpcGetRequestCount": 0, "rpcFullScanRequestCount": 0, "rpcScanRequestCount": 0, "rpcMultiRequestCount": 0, "rpcMutateRequestCount": 0, "checkMutateFailedCount": 0, "checkMutatePassedCount": 0, "blockCacheHitCount": 0, "blockCacheHitCountPrimary": 0, "blockCacheHitCachingCount": 0, "blockCacheMissCount": 0, "blockCacheMissCountPrimary": 0, "blockCacheMissCachingCount": 0, "blockCacheEvictionCount": 0, "blockCacheEvictionCountPrimary": 0, "blockCacheFailedInsertionCount": 0, "blockCacheDataMissCount": 0, "blockCacheLeafIndexMissCount": 0, "blockCacheBloomChunkMissCount": 0, "blockCacheMetaMissCount": 0, "blockCacheRootIndexMissCount": 0, "blockCacheIntermediateIndexMissCount": 0, "blockCacheFileInfoMissCount": 0, "blockCacheGeneralBloomMetaMissCount": 0, "blockCacheDeleteFamilyBloomMissCount": 0, "blockCacheTrailerMissCount": 0, "blockCacheDataHitCount": 0, "blockCacheLeafIndexHitCount": 0, "blockCacheBloomChunkHitCount": 0, "blockCacheMetaHitCount": 0, "blockCacheRootIndexHitCount": 0, "blockCacheIntermediateIndexHitCount": 0, "blockCacheFileInfoHitCount": 0, "blockCacheGeneralBloomMetaHitCount": 0, "blockCacheDeleteFamilyBloomHitCount": 0, "blockCacheTrailerHitCount": 0, "updatesBlockedTime": 0, "flushedCellsCount": 0, "compactedCellsCount": 0, "majorCompactedCellsCount": 0, "flushedCellsSize": 0, "compactedCellsSize": 0, "majorCompactedCellsSize": 0, "cellsCountCompactedFromMob": 0, "cellsCountCompactedToMob": 0, "cellsSizeCompactedFromMob": 0, "cellsSizeCompactedToMob": 0, "mobFlushCount": 0, "mobFlushedCellsCount": 0, "mobFlushedCellsSize": 0, "mobScanCellsCount": 0, "mobScanCellsSize": 0, "mobFileCacheAccessCount": 0, "mobFileCacheMissCount": 0, "mobFileCacheEvictedCount": 0, "hedgedReads": 0, "hedgedReadWins": 0, "hedgedReadOpsInCurThread": 0, "blockedRequestCount": 0, "CheckAndMutate_num_ops": 0, "CheckAndMutate_min": 0, "CheckAndMutate_max": 0, "CheckAndMutate_mean": 0, "CheckAndMutate_25th_percentile": 0, "CheckAndMutate_median": 0, "CheckAndMutate_75th_percentile": 0, "CheckAndMutate_90th_percentile": 0, "CheckAndMutate_95th_percentile": 0, "CheckAndMutate_98th_percentile": 0, "CheckAndMutate_99th_percentile": 0, "CheckAndMutate_99.9th_percentile": 0, "MajorCompactionTime_num_ops": 0, "MajorCompactionTime_min": 0, "MajorCompactionTime_max": 0, "MajorCompactionTime_mean": 0, "MajorCompactionTime_25th_percentile": 0, "MajorCompactionTime_median": 0, "MajorCompactionTime_75th_percentile": 0, "MajorCompactionTime_90th_percentile": 0, "MajorCompactionTime_95th_percentile": 0, "MajorCompactionTime_98th_percentile": 0, "MajorCompactionTime_99th_percentile": 0, "MajorCompactionTime_99.9th_percentile": 0, "ScanTime_num_ops": 0, "ScanTime_min": 0, "ScanTime_max": 0, "ScanTime_mean": 0, "ScanTime_25th_percentile": 0, "ScanTime_median": 0, "ScanTime_75th_percentile": 0, "ScanTime_90th_percentile": 0, "ScanTime_95th_percentile": 0, "ScanTime_98th_percentile": 0, "ScanTime_99th_percentile": 0, "ScanTime_99.9th_percentile": 0, "CheckAndMutateBlockBytesScanned_num_ops": 0, "CheckAndMutateBlockBytesScanned_min": 0, "CheckAndMutateBlockBytesScanned_max": 0, "CheckAndMutateBlockBytesScanned_mean": 0, "CheckAndMutateBlockBytesScanned_25th_percentile": 0, "CheckAndMutateBlockBytesScanned_median": 0, "CheckAndMutateBlockBytesScanned_75th_percentile": 0, "CheckAndMutateBlockBytesScanned_90th_percentile": 0, "CheckAndMutateBlockBytesScanned_95th_percentile": 0, "CheckAndMutateBlockBytesScanned_98th_percentile": 0, "CheckAndMutateBlockBytesScanned_99th_percentile": 0, "CheckAndMutateBlockBytesScanned_99.9th_percentile": 0, "Put_num_ops": 0, "Put_min": 0, "Put_max": 0, "Put_mean": 0, "Put_25th_percentile": 0, "Put_median": 0, "Put_75th_percentile": 0, "Put_90th_percentile": 0, "Put_95th_percentile": 0, "Put_98th_percentile": 0, "Put_99th_percentile": 0, "Put_99.9th_percentile": 0, "splitRequestCount": 0, "AppendBlockBytesScanned_num_ops": 0, "AppendBlockBytesScanned_min": 0, "AppendBlockBytesScanned_max": 0, "AppendBlockBytesScanned_mean": 0, "AppendBlockBytesScanned_25th_percentile": 0, "AppendBlockBytesScanned_median": 0, "AppendBlockBytesScanned_75th_percentile": 0, "AppendBlockBytesScanned_90th_percentile": 0, "AppendBlockBytesScanned_95th_percentile": 0, "AppendBlockBytesScanned_98th_percentile": 0, "AppendBlockBytesScanned_99th_percentile": 0, "AppendBlockBytesScanned_99.9th_percentile": 0, "PutBatch_num_ops": 0, "PutBatch_min": 0, "PutBatch_max": 0, "PutBatch_mean": 0, "PutBatch_25th_percentile": 0, "PutBatch_median": 0, "PutBatch_75th_percentile": 0, "PutBatch_90th_percentile": 0, "PutBatch_95th_percentile": 0, "PutBatch_98th_percentile": 0, "PutBatch_99th_percentile": 0, "PutBatch_99.9th_percentile": 0, "IncrementBlockBytesScanned_num_ops": 0, "IncrementBlockBytesScanned_min": 0, "IncrementBlockBytesScanned_max": 0, "IncrementBlockBytesScanned_mean": 0, "IncrementBlockBytesScanned_25th_percentile": 0, "IncrementBlockBytesScanned_median": 0, "IncrementBlockBytesScanned_75th_percentile": 0, "IncrementBlockBytesScanned_90th_percentile": 0, "IncrementBlockBytesScanned_95th_percentile": 0, "IncrementBlockBytesScanned_98th_percentile": 0, "IncrementBlockBytesScanned_99th_percentile": 0, "IncrementBlockBytesScanned_99.9th_percentile": 0, "SplitTime_num_ops": 0, "SplitTime_min": 0, "SplitTime_max": 0, "SplitTime_mean": 0, "SplitTime_25th_percentile": 0, "SplitTime_median": 0, "SplitTime_75th_percentile": 0, "SplitTime_90th_percentile": 0, "SplitTime_95th_percentile": 0, "SplitTime_98th_percentile": 0, "SplitTime_99th_percentile": 0, "SplitTime_99.9th_percentile": 0, "GetBlockBytesScanned_num_ops": 0, "GetBlockBytesScanned_min": 0, "GetBlockBytesScanned_max": 0, "GetBlockBytesScanned_mean": 0, "GetBlockBytesScanned_25th_percentile": 0, "GetBlockBytesScanned_median": 0, "GetBlockBytesScanned_75th_percentile": 0, "GetBlockBytesScanned_90th_percentile": 0, "GetBlockBytesScanned_95th_percentile": 0, "GetBlockBytesScanned_98th_percentile": 0, "GetBlockBytesScanned_99th_percentile": 0, "GetBlockBytesScanned_99.9th_percentile": 0, "majorCompactedInputBytes": 0, "slowAppendCount": 0, "flushedOutputBytes": 0, "Replay_num_ops": 0, "Replay_min": 0, "Replay_max": 0, "Replay_mean": 0, "Replay_25th_percentile": 0, "Replay_median": 0, "Replay_75th_percentile": 0, "Replay_90th_percentile": 0, "Replay_95th_percentile": 0, "Replay_98th_percentile": 0, "Replay_99th_percentile": 0, "Replay_99.9th_percentile": 0, "MajorCompactionInputSize_num_ops": 0, "MajorCompactionInputSize_min": 0, "MajorCompactionInputSize_max": 0, "MajorCompactionInputSize_mean": 0, "MajorCompactionInputSize_25th_percentile": 0, "MajorCompactionInputSize_median": 0, "MajorCompactionInputSize_75th_percentile": 0, "MajorCompactionInputSize_90th_percentile": 0, "MajorCompactionInputSize_95th_percentile": 0, "MajorCompactionInputSize_98th_percentile": 0, "MajorCompactionInputSize_99th_percentile": 0, "MajorCompactionInputSize_99.9th_percentile": 0, "pauseInfoThresholdExceeded": 0, "CheckAndDelete_num_ops": 0, "CheckAndDelete_min": 0, "CheckAndDelete_max": 0, "CheckAndDelete_mean": 0, "CheckAndDelete_25th_percentile": 0, "CheckAndDelete_median": 0, "CheckAndDelete_75th_percentile": 0, "CheckAndDelete_90th_percentile": 0, "CheckAndDelete_95th_percentile": 0, "CheckAndDelete_98th_percentile": 0, "CheckAndDelete_99th_percentile": 0, "CheckAndDelete_99.9th_percentile": 0, "CompactionInputSize_num_ops": 0, "CompactionInputSize_min": 0, "CompactionInputSize_max": 0, "CompactionInputSize_mean": 0, "CompactionInputSize_25th_percentile": 0, "CompactionInputSize_median": 0, "CompactionInputSize_75th_percentile": 0, "CompactionInputSize_90th_percentile": 0, "CompactionInputSize_95th_percentile": 0, "CompactionInputSize_98th_percentile": 0, "CompactionInputSize_99th_percentile": 0, "CompactionInputSize_99.9th_percentile": 0, "flushedMemstoreBytes": 0, "majorCompactedOutputBytes": 0, "slowPutCount": 0, "compactedInputBytes": 0, "FlushOutputSize_num_ops": 0, "FlushOutputSize_min": 0, "FlushOutputSize_max": 0, "FlushOutputSize_mean": 0, "FlushOutputSize_25th_percentile": 0, "FlushOutputSize_median": 0, "FlushOutputSize_75th_percentile": 0, "FlushOutputSize_90th_percentile": 0, "FlushOutputSize_95th_percentile": 0, "FlushOutputSize_98th_percentile": 0, "FlushOutputSize_99th_percentile": 0, "FlushOutputSize_99.9th_percentile": 0, "PauseTimeWithGc_num_ops": 0, "PauseTimeWithGc_min": 0, "PauseTimeWithGc_max": 0, "PauseTimeWithGc_mean": 0, "PauseTimeWithGc_25th_percentile": 0, "PauseTimeWithGc_median": 0, "PauseTimeWithGc_75th_percentile": 0, "PauseTimeWithGc_90th_percentile": 0, "PauseTimeWithGc_95th_percentile": 0, "PauseTimeWithGc_98th_percentile": 0, "PauseTimeWithGc_99th_percentile": 0, "PauseTimeWithGc_99.9th_percentile": 0, "compactedOutputBytes": 0, "pauseWarnThresholdExceeded": 0, "ScanBlockBytesScanned_num_ops": 0, "ScanBlockBytesScanned_min": 0, "ScanBlockBytesScanned_max": 0, "ScanBlockBytesScanned_mean": 0, "ScanBlockBytesScanned_25th_percentile": 0, "ScanBlockBytesScanned_median": 0, "ScanBlockBytesScanned_75th_percentile": 0, "ScanBlockBytesScanned_90th_percentile": 0, "ScanBlockBytesScanned_95th_percentile": 0, "ScanBlockBytesScanned_98th_percentile": 0, "ScanBlockBytesScanned_99th_percentile": 0, "ScanBlockBytesScanned_99.9th_percentile": 0, "Increment_num_ops": 0, "Increment_min": 0, "Increment_max": 0, "Increment_mean": 0, "Increment_25th_percentile": 0, "Increment_median": 0, "Increment_75th_percentile": 0, "Increment_90th_percentile": 0, "Increment_95th_percentile": 0, "Increment_98th_percentile": 0, "Increment_99th_percentile": 0, "Increment_99.9th_percentile": 0, "Delete_num_ops": 0, "Delete_min": 0, "Delete_max": 0, "Delete_mean": 0, "Delete_25th_percentile": 0, "Delete_median": 0, "Delete_75th_percentile": 0, "Delete_90th_percentile": 0, "Delete_95th_percentile": 0, "Delete_98th_percentile": 0, "Delete_99th_percentile": 0, "Delete_99.9th_percentile": 0, "DeleteBatch_num_ops": 0, "DeleteBatch_min": 0, "DeleteBatch_max": 0, "DeleteBatch_mean": 0, "DeleteBatch_25th_percentile": 0, "DeleteBatch_median": 0, "DeleteBatch_75th_percentile": 0, "DeleteBatch_90th_percentile": 0, "DeleteBatch_95th_percentile": 0, "DeleteBatch_98th_percentile": 0, "DeleteBatch_99th_percentile": 0, "DeleteBatch_99.9th_percentile": 0, "blockBytesScannedCount": 0, "FlushMemstoreSize_num_ops": 0, "FlushMemstoreSize_min": 0, "FlushMemstoreSize_max": 0, "FlushMemstoreSize_mean": 0, "FlushMemstoreSize_25th_percentile": 0, "FlushMemstoreSize_median": 0, "FlushMemstoreSize_75th_percentile": 0, "FlushMemstoreSize_90th_percentile": 0, "FlushMemstoreSize_95th_percentile": 0, "FlushMemstoreSize_98th_percentile": 0, "FlushMemstoreSize_99th_percentile": 0, "FlushMemstoreSize_99.9th_percentile": 0, "CompactionInputFileCount_num_ops": 0, "CompactionInputFileCount_min": 0, "CompactionInputFileCount_max": 0, "CompactionInputFileCount_mean": 0, "CompactionInputFileCount_25th_percentile": 0, "CompactionInputFileCount_median": 0, "CompactionInputFileCount_75th_percentile": 0, "CompactionInputFileCount_90th_percentile": 0, "CompactionInputFileCount_95th_percentile": 0, "CompactionInputFileCount_98th_percentile": 0, "CompactionInputFileCount_99th_percentile": 0, "CompactionInputFileCount_99.9th_percentile": 0, "CompactionTime_num_ops": 0, "CompactionTime_min": 0, "CompactionTime_max": 0, "CompactionTime_mean": 0, "CompactionTime_25th_percentile": 0, "CompactionTime_median": 0, "CompactionTime_75th_percentile": 0, "CompactionTime_90th_percentile": 0, "CompactionTime_95th_percentile": 0, "CompactionTime_98th_percentile": 0, "CompactionTime_99th_percentile": 0, "CompactionTime_99.9th_percentile": 0, "Get_num_ops": 0, "Get_min": 0, "Get_max": 0, "Get_mean": 0, "Get_25th_percentile": 0, "Get_median": 0, "Get_75th_percentile": 0, "Get_90th_percentile": 0, "Get_95th_percentile": 0, "Get_98th_percentile": 0, "Get_99th_percentile": 0, "Get_99.9th_percentile": 0, "MajorCompactionInputFileCount_num_ops": 0, "MajorCompactionInputFileCount_min": 0, "MajorCompactionInputFileCount_max": 0, "MajorCompactionInputFileCount_mean": 0, "MajorCompactionInputFileCount_25th_percentile": 0, "MajorCompactionInputFileCount_median": 0, "MajorCompactionInputFileCount_75th_percentile": 0, "MajorCompactionInputFileCount_90th_percentile": 0, "MajorCompactionInputFileCount_95th_percentile": 0, "MajorCompactionInputFileCount_98th_percentile": 0, "MajorCompactionInputFileCount_99th_percentile": 0, "MajorCompactionInputFileCount_99.9th_percentile": 0, "scannerLeaseExpiredCount": 0, "CheckAndPut_num_ops": 0, "CheckAndPut_min": 0, "CheckAndPut_max": 0, "CheckAndPut_mean": 0, "CheckAndPut_25th_percentile": 0, "CheckAndPut_median": 0, "CheckAndPut_75th_percentile": 0, "CheckAndPut_90th_percentile": 0, "CheckAndPut_95th_percentile": 0, "CheckAndPut_98th_percentile": 0, "CheckAndPut_99th_percentile": 0, "CheckAndPut_99.9th_percentile": 0, "MajorCompactionOutputSize_num_ops": 0, "MajorCompactionOutputSize_min": 0, "MajorCompactionOutputSize_max": 0, "MajorCompactionOutputSize_mean": 0, "MajorCompactionOutputSize_25th_percentile": 0, "MajorCompactionOutputSize_median": 0, "MajorCompactionOutputSize_75th_percentile": 0, "MajorCompactionOutputSize_90th_percentile": 0, "MajorCompactionOutputSize_95th_percentile": 0, "MajorCompactionOutputSize_98th_percentile": 0, "MajorCompactionOutputSize_99th_percentile": 0, "MajorCompactionOutputSize_99.9th_percentile": 0, "CompactionOutputFileCount_num_ops": 0, "CompactionOutputFileCount_min": 0, "CompactionOutputFileCount_max": 0, "CompactionOutputFileCount_mean": 0, "CompactionOutputFileCount_25th_percentile": 0, "CompactionOutputFileCount_median": 0, "CompactionOutputFileCount_75th_percentile": 0, "CompactionOutputFileCount_90th_percentile": 0, "CompactionOutputFileCount_95th_percentile": 0, "CompactionOutputFileCount_98th_percentile": 0, "CompactionOutputFileCount_99th_percentile": 0, "CompactionOutputFileCount_99.9th_percentile": 0, "slowDeleteCount": 0, "FlushTime_num_ops": 0, "FlushTime_min": 0, "FlushTime_max": 0, "FlushTime_mean": 0, "FlushTime_25th_percentile": 0, "FlushTime_median": 0, "FlushTime_75th_percentile": 0, "FlushTime_90th_percentile": 0, "FlushTime_95th_percentile": 0, "FlushTime_98th_percentile": 0, "FlushTime_99th_percentile": 0, "FlushTime_99.9th_percentile": 0, "splitSuccessCount": 0, "MajorCompactionOutputFileCount_num_ops": 0, "MajorCompactionOutputFileCount_min": 0, "MajorCompactionOutputFileCount_max": 0, "MajorCompactionOutputFileCount_mean": 0, "MajorCompactionOutputFileCount_25th_percentile": 0, "MajorCompactionOutputFileCount_median": 0, "MajorCompactionOutputFileCount_75th_percentile": 0, "MajorCompactionOutputFileCount_90th_percentile": 0, "MajorCompactionOutputFileCount_95th_percentile": 0, "MajorCompactionOutputFileCount_98th_percentile": 0, "MajorCompactionOutputFileCount_99th_percentile": 0, "MajorCompactionOutputFileCount_99.9th_percentile": 0, "slowGetCount": 0, "ScanSize_num_ops": 0, "ScanSize_min": 0, "ScanSize_max": 0, "ScanSize_mean": 0, "ScanSize_25th_percentile": 0, "ScanSize_median": 0, "ScanSize_75th_percentile": 0, "ScanSize_90th_percentile": 0, "ScanSize_95th_percentile": 0, "ScanSize_98th_percentile": 0, "ScanSize_99th_percentile": 0, "ScanSize_99.9th_percentile": 0, "CompactionOutputSize_num_ops": 0, "CompactionOutputSize_min": 0, "CompactionOutputSize_max": 0, "CompactionOutputSize_mean": 0, "CompactionOutputSize_25th_percentile": 0, "CompactionOutputSize_median": 0, "CompactionOutputSize_75th_percentile": 0, "CompactionOutputSize_90th_percentile": 0, "CompactionOutputSize_95th_percentile": 0, "CompactionOutputSize_98th_percentile": 0, "CompactionOutputSize_99th_percentile": 0, "CompactionOutputSize_99.9th_percentile": 0, "PauseTimeWithoutGc_num_ops": 0, "PauseTimeWithoutGc_min": 0, "PauseTimeWithoutGc_max": 0, "PauseTimeWithoutGc_mean": 0, "PauseTimeWithoutGc_25th_percentile": 0, "PauseTimeWithoutGc_median": 0, "PauseTimeWithoutGc_75th_percentile": 0, "PauseTimeWithoutGc_90th_percentile": 0, "PauseTimeWithoutGc_95th_percentile": 0, "PauseTimeWithoutGc_98th_percentile": 0, "PauseTimeWithoutGc_99th_percentile": 0, "PauseTimeWithoutGc_99.9th_percentile": 0, "slowIncrementCount": 0, "Append_num_ops": 0, "Append_min": 0, "Append_max": 0, "Append_mean": 0, "Append_25th_percentile": 0, "Append_median": 0, "Append_75th_percentile": 0, "Append_90th_percentile": 0, "Append_95th_percentile": 0, "Append_98th_percentile": 0, "Append_99th_percentile": 0, "Append_99.9th_percentile": 0, "Bulkload_count": 0, "Bulkload_mean_rate": 0.0, "Bulkload_1min_rate": 0.0, "Bulkload_5min_rate": 0.0, "Bulkload_15min_rate": 0.0, "Bulkload_num_ops": 0, "Bulkload_min": 0, "Bulkload_max": 0, "Bulkload_mean": 0, "Bulkload_25th_percentile": 0, "Bulkload_median": 0, "Bulkload_75th_percentile": 0, "Bulkload_90th_percentile": 0, "Bulkload_95th_percentile": 0, "Bulkload_98th_percentile": 0, "Bulkload_99th_percentile": 0, "Bulkload_99.9th_percentile": 0 } ] } 2024-11-24T07:35:30,364 WARN [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37719 {}] master.MasterRpcServices(700): 1ab588ae992e,40611,1732433713633 reported a fatal error: ***** ABORTING region server 1ab588ae992e,40611,1732433713633: testing ***** 2024-11-24T07:35:30,365 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '1ab588ae992e,40611,1732433713633' ***** 2024-11-24T07:35:30,366 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: testing 2024-11-24T07:35:30,366 INFO [RS:2;1ab588ae992e:40611 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-24T07:35:30,366 INFO [RS:2;1ab588ae992e:40611 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager abruptly. 2024-11-24T07:35:30,366 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-24T07:35:30,366 INFO [RS:2;1ab588ae992e:40611 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager abruptly. 2024-11-24T07:35:30,366 INFO [RS:2;1ab588ae992e:40611 {}] regionserver.HRegionServer(3091): Received CLOSE for 567d9cab2ff71509e601db9c0fa44478 2024-11-24T07:35:30,367 INFO [RS:2;1ab588ae992e:40611 {}] regionserver.HRegionServer(956): aborting server 1ab588ae992e,40611,1732433713633 2024-11-24T07:35:30,367 INFO [RS:2;1ab588ae992e:40611 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-24T07:35:30,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46433 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Get size: 140 connection: 172.17.0.3:41970 deadline: 1732433790367, exception=org.apache.hadoop.hbase.exceptions.RegionMovedException: Region moved to: hostname=1ab588ae992e port=40611 startCode=1732433713633. As of locationSeqNum=12. 2024-11-24T07:35:30,367 INFO [RS:2;1ab588ae992e:40611 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:2;1ab588ae992e:40611. 2024-11-24T07:35:30,367 DEBUG [RS_CLOSE_REGION-regionserver/1ab588ae992e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 567d9cab2ff71509e601db9c0fa44478, disabling compactions & flushes 2024-11-24T07:35:30,367 DEBUG [RS:2;1ab588ae992e:40611 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-24T07:35:30,367 INFO [RS_CLOSE_REGION-regionserver/1ab588ae992e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testReplayEditsAfterRegionMovedWithMultiCF,,1732433728060.567d9cab2ff71509e601db9c0fa44478. 2024-11-24T07:35:30,367 DEBUG [RS_CLOSE_REGION-regionserver/1ab588ae992e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1732433728060.567d9cab2ff71509e601db9c0fa44478. 2024-11-24T07:35:30,367 DEBUG [RS_CLOSE_REGION-regionserver/1ab588ae992e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1732433728060.567d9cab2ff71509e601db9c0fa44478. after waiting 0 ms 2024-11-24T07:35:30,367 DEBUG [RPCClient-NioEventLoopGroup-6-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1732433728060.567d9cab2ff71509e601db9c0fa44478., hostname=1ab588ae992e,46433,1732433713473, seqNum=5 , the old value is region=testReplayEditsAfterRegionMovedWithMultiCF,,1732433728060.567d9cab2ff71509e601db9c0fa44478., hostname=1ab588ae992e,46433,1732433713473, seqNum=5, error=org.apache.hadoop.hbase.exceptions.RegionMovedException: Region moved to: hostname=1ab588ae992e port=40611 startCode=1732433713633. As of locationSeqNum=12. 2024-11-24T07:35:30,367 DEBUG [RS_CLOSE_REGION-regionserver/1ab588ae992e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsAfterRegionMovedWithMultiCF,,1732433728060.567d9cab2ff71509e601db9c0fa44478. 2024-11-24T07:35:30,367 DEBUG [RS:2;1ab588ae992e:40611 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T07:35:30,367 DEBUG [RPCClient-NioEventLoopGroup-6-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1732433728060.567d9cab2ff71509e601db9c0fa44478., hostname=1ab588ae992e,46433,1732433713473, seqNum=5 is org.apache.hadoop.hbase.exceptions.RegionMovedException: Region moved to: hostname=1ab588ae992e port=40611 startCode=1732433713633. As of locationSeqNum=12. 2024-11-24T07:35:30,368 DEBUG [RPCClient-NioEventLoopGroup-6-6 {}] client.AsyncRegionLocatorHelper(84): Try updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1732433728060.567d9cab2ff71509e601db9c0fa44478., hostname=1ab588ae992e,46433,1732433713473, seqNum=5 with the new location region=testReplayEditsAfterRegionMovedWithMultiCF,,1732433728060.567d9cab2ff71509e601db9c0fa44478., hostname=1ab588ae992e,40611,1732433713633, seqNum=12 constructed by org.apache.hadoop.hbase.exceptions.RegionMovedException: Region moved to: hostname=1ab588ae992e port=40611 startCode=1732433713633. As of locationSeqNum=12. 2024-11-24T07:35:30,368 INFO [RS:2;1ab588ae992e:40611 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-24T07:35:30,368 INFO [RS:2;1ab588ae992e:40611 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-24T07:35:30,368 INFO [RS:2;1ab588ae992e:40611 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-24T07:35:30,368 INFO [RS:2;1ab588ae992e:40611 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-11-24T07:35:30,369 INFO [RS:2;1ab588ae992e:40611 {}] regionserver.HRegionServer(1321): Waiting on 2 regions to close 2024-11-24T07:35:30,369 DEBUG [RS_CLOSE_META-regionserver/1ab588ae992e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-24T07:35:30,369 DEBUG [RS:2;1ab588ae992e:40611 {}] regionserver.HRegionServer(1325): Online Regions={1588230740=hbase:meta,,1.1588230740, 567d9cab2ff71509e601db9c0fa44478=testReplayEditsAfterRegionMovedWithMultiCF,,1732433728060.567d9cab2ff71509e601db9c0fa44478.} 2024-11-24T07:35:30,369 INFO [RS_CLOSE_META-regionserver/1ab588ae992e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-24T07:35:30,369 DEBUG [RS_CLOSE_META-regionserver/1ab588ae992e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-24T07:35:30,369 DEBUG [RS_CLOSE_META-regionserver/1ab588ae992e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-24T07:35:30,369 DEBUG [RS_CLOSE_META-regionserver/1ab588ae992e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-24T07:35:30,369 DEBUG [RS:2;1ab588ae992e:40611 {}] regionserver.HRegionServer(1351): Waiting on 1588230740, 567d9cab2ff71509e601db9c0fa44478 2024-11-24T07:35:30,369 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.regionserver.RegionServerAbortedException: Server 1ab588ae992e,40611,1732433713633 aborting at org.apache.hadoop.hbase.ipc.ServerRpcConnection.processRequest(ServerRpcConnection.java:564) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.ServerRpcConnection.processOneRpc(ServerRpcConnection.java:364) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyServerRpcConnection.process(NettyServerRpcConnection.java:89) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcServerRequestDecoder.channelRead0(NettyRpcServerRequestDecoder.java:56) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcServerRequestDecoder.channelRead0(NettyRpcServerRequestDecoder.java:31) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollStreamChannel$EpollStreamUnsafe.epollInReady(AbstractEpollStreamChannel.java:799) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:501) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T07:35:30,369 INFO [RS_CLOSE_REGION-regionserver/1ab588ae992e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testReplayEditsAfterRegionMovedWithMultiCF,,1732433728060.567d9cab2ff71509e601db9c0fa44478. 2024-11-24T07:35:30,370 DEBUG [RS_CLOSE_REGION-regionserver/1ab588ae992e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 567d9cab2ff71509e601db9c0fa44478: Waiting for close lock at 1732433730367Running coprocessor pre-close hooks at 1732433730367Disabling compacts and flushes for region at 1732433730367Disabling writes for close at 1732433730367Writing region close event to WAL at 1732433730369 (+2 ms)Running coprocessor post-close hooks at 1732433730369Closed at 1732433730369 2024-11-24T07:35:30,370 DEBUG [RS_CLOSE_REGION-regionserver/1ab588ae992e:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testReplayEditsAfterRegionMovedWithMultiCF,,1732433728060.567d9cab2ff71509e601db9c0fa44478. 2024-11-24T07:35:30,370 ERROR [RS_CLOSE_META-regionserver/1ab588ae992e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1960): Memstore data size is 5811 in region hbase:meta,,1.1588230740 2024-11-24T07:35:30,370 DEBUG [RS_CLOSE_META-regionserver/1ab588ae992e:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-24T07:35:30,370 INFO [RS_CLOSE_META-regionserver/1ab588ae992e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-24T07:35:30,370 DEBUG [RS_CLOSE_META-regionserver/1ab588ae992e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1732433730369Running coprocessor pre-close hooks at 1732433730369Disabling compacts and flushes for region at 1732433730369Disabling writes for close at 1732433730369Writing region close event to WAL at 1732433730370 (+1 ms)Running coprocessor post-close hooks at 1732433730370Closed at 1732433730370 2024-11-24T07:35:30,370 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=1ab588ae992e,40611,1732433713633, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=1ab588ae992e,40611,1732433713633, seqNum=-1, error=org.apache.hadoop.hbase.regionserver.RegionServerAbortedException: Server 1ab588ae992e,40611,1732433713633 aborting 2024-11-24T07:35:30,370 DEBUG [RS_CLOSE_META-regionserver/1ab588ae992e:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-11-24T07:35:30,370 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:meta,,1.1588230740, hostname=1ab588ae992e,40611,1732433713633, seqNum=-1 is org.apache.hadoop.hbase.regionserver.RegionServerAbortedException: Server 1ab588ae992e,40611,1732433713633 aborting 2024-11-24T07:35:30,371 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:meta,,1.1588230740, hostname=1ab588ae992e,40611,1732433713633, seqNum=-1 from cache 2024-11-24T07:35:30,440 INFO [regionserver/1ab588ae992e:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-24T07:35:30,472 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-24T07:35:30,474 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=1ab588ae992e,40611,1732433713633, seqNum=-1] 2024-11-24T07:35:30,475 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.regionserver.RegionServerAbortedException: Server 1ab588ae992e,40611,1732433713633 aborting at org.apache.hadoop.hbase.ipc.ServerRpcConnection.processRequest(ServerRpcConnection.java:564) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.ServerRpcConnection.processOneRpc(ServerRpcConnection.java:364) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyServerRpcConnection.process(NettyServerRpcConnection.java:89) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcServerRequestDecoder.channelRead0(NettyRpcServerRequestDecoder.java:56) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcServerRequestDecoder.channelRead0(NettyRpcServerRequestDecoder.java:31) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollStreamChannel$EpollStreamUnsafe.epollInReady(AbstractEpollStreamChannel.java:799) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:501) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T07:35:30,477 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=1ab588ae992e,40611,1732433713633, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=1ab588ae992e,40611,1732433713633, seqNum=-1, error=org.apache.hadoop.hbase.regionserver.RegionServerAbortedException: Server 1ab588ae992e,40611,1732433713633 aborting 2024-11-24T07:35:30,477 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:meta,,1.1588230740, hostname=1ab588ae992e,40611,1732433713633, seqNum=-1 is org.apache.hadoop.hbase.regionserver.RegionServerAbortedException: Server 1ab588ae992e,40611,1732433713633 aborting 2024-11-24T07:35:30,477 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:meta,,1.1588230740, hostname=1ab588ae992e,40611,1732433713633, seqNum=-1 from cache 2024-11-24T07:35:30,570 INFO [RS:2;1ab588ae992e:40611 {}] regionserver.HRegionServer(976): stopping server 1ab588ae992e,40611,1732433713633; all regions closed. 2024-11-24T07:35:30,578 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42685 is added to blk_1073741836_1012 (size=3556) 2024-11-24T07:35:30,578 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41161 is added to blk_1073741836_1012 (size=3556) 2024-11-24T07:35:30,578 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41971 is added to blk_1073741836_1012 (size=3556) 2024-11-24T07:35:30,582 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41971 is added to blk_1073741835_1011 (size=1405) 2024-11-24T07:35:30,583 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42685 is added to blk_1073741835_1011 (size=1405) 2024-11-24T07:35:30,583 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41161 is added to blk_1073741835_1011 (size=1405) 2024-11-24T07:35:30,584 DEBUG [RS:2;1ab588ae992e:40611 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T07:35:30,584 INFO [RS:2;1ab588ae992e:40611 {}] regionserver.LeaseManager(133): Closed leases 2024-11-24T07:35:30,584 INFO [RS:2;1ab588ae992e:40611 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-24T07:35:30,585 INFO [RS:2;1ab588ae992e:40611 {}] hbase.ChoreService(370): Chore service for: regionserver/1ab588ae992e:0 had [ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS] on shutdown 2024-11-24T07:35:30,585 INFO [RS:2;1ab588ae992e:40611 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-24T07:35:30,585 INFO [regionserver/1ab588ae992e:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-24T07:35:30,585 INFO [RS:2;1ab588ae992e:40611 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.3:40611 2024-11-24T07:35:30,645 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40611-0x1016bc48eea0003, quorum=127.0.0.1:49244, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/1ab588ae992e,40611,1732433713633 2024-11-24T07:35:30,645 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37719-0x1016bc48eea0000, quorum=127.0.0.1:49244, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-24T07:35:30,645 INFO [RS:2;1ab588ae992e:40611 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-24T07:35:30,646 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [1ab588ae992e,40611,1732433713633] 2024-11-24T07:35:30,666 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/1ab588ae992e,40611,1732433713633 already deleted, retry=false 2024-11-24T07:35:30,666 INFO [RegionServerTracker-0 {}] master.ServerManager(695): Processing expiration of 1ab588ae992e,40611,1732433713633 on 1ab588ae992e,37719,1732433712713 2024-11-24T07:35:30,670 DEBUG [RegionServerTracker-0 {}] procedure2.ProcedureExecutor(1139): Stored pid=13, state=RUNNABLE:SERVER_CRASH_START, hasLock=false; ServerCrashProcedure 1ab588ae992e,40611,1732433713633, splitWal=true, meta=true 2024-11-24T07:35:30,673 INFO [RegionServerTracker-0 {}] assignment.AssignmentManager(1999): Scheduled ServerCrashProcedure pid=13 for 1ab588ae992e,40611,1732433713633 (carryingMeta=true) 1ab588ae992e,40611,1732433713633/CRASHED/regionCount=2/lock=java.util.concurrent.locks.ReentrantReadWriteLock@45371c47[Write locks = 1, Read locks = 0], oldState=ONLINE. 2024-11-24T07:35:30,674 INFO [PEWorker-2 {}] procedure.ServerCrashProcedure(169): Start pid=13, state=RUNNABLE:SERVER_CRASH_START, hasLock=true; ServerCrashProcedure 1ab588ae992e,40611,1732433713633, splitWal=true, meta=true 2024-11-24T07:35:30,675 INFO [PEWorker-2 {}] procedure.ServerCrashProcedure(339): Splitting WALs pid=13, state=RUNNABLE:SERVER_CRASH_SPLIT_META_LOGS, hasLock=true; ServerCrashProcedure 1ab588ae992e,40611,1732433713633, splitWal=true, meta=true, isMeta: true 2024-11-24T07:35:30,676 DEBUG [PEWorker-2 {}] master.MasterWalManager(329): Renamed region directory: hdfs://localhost:33505/user/jenkins/test-data/e37915a4-8313-e328-91fb-37d541e8b214/WALs/1ab588ae992e,40611,1732433713633-splitting 2024-11-24T07:35:30,677 INFO [PEWorker-2 {}] master.SplitWALManager(105): 1ab588ae992e,40611,1732433713633 WAL count=1, meta=true 2024-11-24T07:35:30,679 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=14, ppid=13, state=RUNNABLE:ACQUIRE_SPLIT_WAL_WORKER, hasLock=false; SplitWALProcedure 1ab588ae992e%2C40611%2C1732433713633.meta.1732433715726.meta}] 2024-11-24T07:35:30,682 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-24T07:35:30,684 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=1ab588ae992e,40611,1732433713633, seqNum=-1] 2024-11-24T07:35:30,684 DEBUG [PEWorker-3 {}] master.SplitWALManager(158): Acquired split WAL worker=1ab588ae992e,46433,1732433713473 2024-11-24T07:35:30,686 WARN [RPCClient-NioEventLoopGroup-6-3 {}] ipc.NettyRpcConnection$2(409): Exception encountered while connecting to the server 1ab588ae992e:40611 org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: Connection refused: 1ab588ae992e/172.17.0.3:40611 Caused by: java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hbase.thirdparty.io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(NioSocketChannel.java:336) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:339) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T07:35:30,687 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=1ab588ae992e,40611,1732433713633, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=1ab588ae992e,40611,1732433713633, seqNum=-1, error=java.net.ConnectException: Call to address=1ab588ae992e:40611 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: Connection refused: 1ab588ae992e/172.17.0.3:40611 2024-11-24T07:35:30,687 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:meta,,1.1588230740, hostname=1ab588ae992e,40611,1732433713633, seqNum=-1 is java.net.ConnectException: Connection refused 2024-11-24T07:35:30,687 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=15, ppid=14, state=RUNNABLE, hasLock=false; SplitWALRemoteProcedure 1ab588ae992e%2C40611%2C1732433713633.meta.1732433715726.meta, worker=1ab588ae992e,46433,1732433713473}] 2024-11-24T07:35:30,687 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:meta,,1.1588230740, hostname=1ab588ae992e,40611,1732433713633, seqNum=-1 from cache 2024-11-24T07:35:30,687 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] ipc.FailedServers(52): Added failed server with address 1ab588ae992e:40611 to list caused by org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: Connection refused: 1ab588ae992e/172.17.0.3:40611 2024-11-24T07:35:30,756 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40611-0x1016bc48eea0003, quorum=127.0.0.1:49244, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-24T07:35:30,756 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40611-0x1016bc48eea0003, quorum=127.0.0.1:49244, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-24T07:35:30,757 INFO [RS:2;1ab588ae992e:40611 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-24T07:35:30,757 INFO [RS:2;1ab588ae992e:40611 {}] regionserver.HRegionServer(1031): Exiting; stopping=1ab588ae992e,40611,1732433713633; zookeeper connection closed. 2024-11-24T07:35:30,757 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@5c308eee {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@5c308eee 2024-11-24T07:35:30,847 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46433 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SplitWALCallable, pid=15 2024-11-24T07:35:30,868 INFO [RS_LOG_REPLAY_OPS-regionserver/1ab588ae992e:0-0 {event_type=RS_LOG_REPLAY, pid=15}] wal.WALSplitter(299): Splitting hdfs://localhost:33505/user/jenkins/test-data/e37915a4-8313-e328-91fb-37d541e8b214/WALs/1ab588ae992e,40611,1732433713633-splitting/1ab588ae992e%2C40611%2C1732433713633.meta.1732433715726.meta, size=3.5 K (3556bytes) 2024-11-24T07:35:30,868 INFO [RS_LOG_REPLAY_OPS-regionserver/1ab588ae992e:0-0 {event_type=RS_LOG_REPLAY, pid=15}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:33505/user/jenkins/test-data/e37915a4-8313-e328-91fb-37d541e8b214/WALs/1ab588ae992e,40611,1732433713633-splitting/1ab588ae992e%2C40611%2C1732433713633.meta.1732433715726.meta 2024-11-24T07:35:30,869 INFO [RS_LOG_REPLAY_OPS-regionserver/1ab588ae992e:0-0 {event_type=RS_LOG_REPLAY, pid=15}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:33505/user/jenkins/test-data/e37915a4-8313-e328-91fb-37d541e8b214/WALs/1ab588ae992e,40611,1732433713633-splitting/1ab588ae992e%2C40611%2C1732433713633.meta.1732433715726.meta after 1ms 2024-11-24T07:35:30,871 DEBUG [RS_LOG_REPLAY_OPS-regionserver/1ab588ae992e:0-0 {event_type=RS_LOG_REPLAY, pid=15}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:33505/user/jenkins/test-data/e37915a4-8313-e328-91fb-37d541e8b214/WALs/1ab588ae992e,40611,1732433713633-splitting/1ab588ae992e%2C40611%2C1732433713633.meta.1732433715726.meta: isRecoveredEdits=false, hasTagCompression=true, hasValueCompression=true, valueCompressionType=GZ 2024-11-24T07:35:30,871 INFO [RS_LOG_REPLAY_OPS-regionserver/1ab588ae992e:0-0 {event_type=RS_LOG_REPLAY, pid=15}] wal.WALSplitter(310): Open hdfs://localhost:33505/user/jenkins/test-data/e37915a4-8313-e328-91fb-37d541e8b214/WALs/1ab588ae992e,40611,1732433713633-splitting/1ab588ae992e%2C40611%2C1732433713633.meta.1732433715726.meta took 3ms 2024-11-24T07:35:30,878 DEBUG [RS_LOG_REPLAY_OPS-regionserver/1ab588ae992e:0-0 {event_type=RS_LOG_REPLAY, pid=15}] wal.WALSplitter(352): Last flushed sequenceid for 1588230740: last_flushed_sequence_id: 18446744073709551615 store_sequence_id { family_name: "info" sequence_id: 5 } store_sequence_id { family_name: "ns" sequence_id: 3 } store_sequence_id { family_name: "rep_barrier" sequence_id: 18446744073709551615 } store_sequence_id { family_name: "table" sequence_id: 6 } 2024-11-24T07:35:30,879 DEBUG [RS_LOG_REPLAY_OPS-regionserver/1ab588ae992e:0-0 {event_type=RS_LOG_REPLAY, pid=15}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:33505/user/jenkins/test-data/e37915a4-8313-e328-91fb-37d541e8b214/WALs/1ab588ae992e,40611,1732433713633-splitting/1ab588ae992e%2C40611%2C1732433713633.meta.1732433715726.meta so closing down 2024-11-24T07:35:30,879 DEBUG [RS_LOG_REPLAY_OPS-regionserver/1ab588ae992e:0-0 {event_type=RS_LOG_REPLAY, pid=15}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-24T07:35:30,880 INFO [RS_LOG_REPLAY_OPS-regionserver/1ab588ae992e:0-0-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000004-1ab588ae992e%2C40611%2C1732433713633.meta.1732433715726.meta.temp 2024-11-24T07:35:30,881 INFO [RS_LOG_REPLAY_OPS-regionserver/1ab588ae992e:0-0-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:33505/user/jenkins/test-data/e37915a4-8313-e328-91fb-37d541e8b214/data/hbase/meta/1588230740/recovered.edits/0000000000000000004-1ab588ae992e%2C40611%2C1732433713633.meta.1732433715726.meta.temp 2024-11-24T07:35:30,882 INFO [RS_LOG_REPLAY_OPS-regionserver/1ab588ae992e:0-0 {event_type=RS_LOG_REPLAY, pid=15}] wal.OutputSink(145): 3 split writer threads finished 2024-11-24T07:35:30,890 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41161 is added to blk_1073741892_1070 (size=3342) 2024-11-24T07:35:30,890 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41971 is added to blk_1073741892_1070 (size=3342) 2024-11-24T07:35:30,890 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42685 is added to blk_1073741892_1070 (size=3342) 2024-11-24T07:35:30,892 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:33505/user/jenkins/test-data/e37915a4-8313-e328-91fb-37d541e8b214/data/hbase/meta/1588230740/recovered.edits/0000000000000000004-1ab588ae992e%2C40611%2C1732433713633.meta.1732433715726.meta.temp (wrote 15 edits, skipped 0 edits in 0 ms) 2024-11-24T07:35:30,894 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:33505/user/jenkins/test-data/e37915a4-8313-e328-91fb-37d541e8b214/data/hbase/meta/1588230740/recovered.edits/0000000000000000004-1ab588ae992e%2C40611%2C1732433713633.meta.1732433715726.meta.temp to hdfs://localhost:33505/user/jenkins/test-data/e37915a4-8313-e328-91fb-37d541e8b214/data/hbase/meta/1588230740/recovered.edits/0000000000000000018 2024-11-24T07:35:30,894 INFO [RS_LOG_REPLAY_OPS-regionserver/1ab588ae992e:0-0 {event_type=RS_LOG_REPLAY, pid=15}] wal.WALSplitter(425): Processed 16 edits across 1 Regions in 22 ms; skipped=1; WAL=hdfs://localhost:33505/user/jenkins/test-data/e37915a4-8313-e328-91fb-37d541e8b214/WALs/1ab588ae992e,40611,1732433713633-splitting/1ab588ae992e%2C40611%2C1732433713633.meta.1732433715726.meta, size=3.5 K, length=3556, corrupted=false, cancelled=false 2024-11-24T07:35:30,894 DEBUG [RS_LOG_REPLAY_OPS-regionserver/1ab588ae992e:0-0 {event_type=RS_LOG_REPLAY, pid=15}] wal.WALSplitter(428): Completed split of hdfs://localhost:33505/user/jenkins/test-data/e37915a4-8313-e328-91fb-37d541e8b214/WALs/1ab588ae992e,40611,1732433713633-splitting/1ab588ae992e%2C40611%2C1732433713633.meta.1732433715726.meta, journal: Splitting hdfs://localhost:33505/user/jenkins/test-data/e37915a4-8313-e328-91fb-37d541e8b214/WALs/1ab588ae992e,40611,1732433713633-splitting/1ab588ae992e%2C40611%2C1732433713633.meta.1732433715726.meta, size=3.5 K (3556bytes) at 1732433730868Finishing writing output for hdfs://localhost:33505/user/jenkins/test-data/e37915a4-8313-e328-91fb-37d541e8b214/WALs/1ab588ae992e,40611,1732433713633-splitting/1ab588ae992e%2C40611%2C1732433713633.meta.1732433715726.meta so closing down at 1732433730879 (+11 ms)Creating recovered edits writer path=hdfs://localhost:33505/user/jenkins/test-data/e37915a4-8313-e328-91fb-37d541e8b214/data/hbase/meta/1588230740/recovered.edits/0000000000000000004-1ab588ae992e%2C40611%2C1732433713633.meta.1732433715726.meta.temp at 1732433730881 (+2 ms)3 split writer threads finished at 1732433730882 (+1 ms)Closed recovered edits writer path=hdfs://localhost:33505/user/jenkins/test-data/e37915a4-8313-e328-91fb-37d541e8b214/data/hbase/meta/1588230740/recovered.edits/0000000000000000004-1ab588ae992e%2C40611%2C1732433713633.meta.1732433715726.meta.temp (wrote 15 edits, skipped 0 edits in 0 ms) at 1732433730892 (+10 ms)Rename recovered edits hdfs://localhost:33505/user/jenkins/test-data/e37915a4-8313-e328-91fb-37d541e8b214/data/hbase/meta/1588230740/recovered.edits/0000000000000000004-1ab588ae992e%2C40611%2C1732433713633.meta.1732433715726.meta.temp to hdfs://localhost:33505/user/jenkins/test-data/e37915a4-8313-e328-91fb-37d541e8b214/data/hbase/meta/1588230740/recovered.edits/0000000000000000018 at 1732433730894 (+2 ms)Processed 16 edits across 1 Regions in 22 ms; skipped=1; WAL=hdfs://localhost:33505/user/jenkins/test-data/e37915a4-8313-e328-91fb-37d541e8b214/WALs/1ab588ae992e,40611,1732433713633-splitting/1ab588ae992e%2C40611%2C1732433713633.meta.1732433715726.meta, size=3.5 K, length=3556, corrupted=false, cancelled=false at 1732433730894 2024-11-24T07:35:30,894 DEBUG [RS_LOG_REPLAY_OPS-regionserver/1ab588ae992e:0-0 {event_type=RS_LOG_REPLAY, pid=15}] regionserver.SplitLogWorker(218): Done splitting WAL hdfs://localhost:33505/user/jenkins/test-data/e37915a4-8313-e328-91fb-37d541e8b214/WALs/1ab588ae992e,40611,1732433713633-splitting/1ab588ae992e%2C40611%2C1732433713633.meta.1732433715726.meta 2024-11-24T07:35:30,896 DEBUG [RS_LOG_REPLAY_OPS-regionserver/1ab588ae992e:0-0 {event_type=RS_LOG_REPLAY, pid=15}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=15 2024-11-24T07:35:30,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37719 {}] master.HMaster(4169): Remote procedure done, pid=15 2024-11-24T07:35:30,903 INFO [PEWorker-4 {}] wal.WALSplitUtil(143): Moved hdfs://localhost:33505/user/jenkins/test-data/e37915a4-8313-e328-91fb-37d541e8b214/WALs/1ab588ae992e,40611,1732433713633-splitting/1ab588ae992e%2C40611%2C1732433713633.meta.1732433715726.meta to hdfs://localhost:33505/user/jenkins/test-data/e37915a4-8313-e328-91fb-37d541e8b214/oldWALs 2024-11-24T07:35:30,905 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=15, resume processing ppid=14 2024-11-24T07:35:30,906 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=15, ppid=14, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 1ab588ae992e%2C40611%2C1732433713633.meta.1732433715726.meta, worker=1ab588ae992e,46433,1732433713473 in 216 msec 2024-11-24T07:35:30,907 DEBUG [PEWorker-5 {}] master.SplitWALManager(172): Release split WAL worker=1ab588ae992e,46433,1732433713473 2024-11-24T07:35:30,911 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=14, resume processing ppid=13 2024-11-24T07:35:30,911 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=14, ppid=13, state=SUCCESS, hasLock=false; SplitWALProcedure 1ab588ae992e%2C40611%2C1732433713633.meta.1732433715726.meta, worker=1ab588ae992e,46433,1732433713473 in 230 msec 2024-11-24T07:35:30,913 INFO [PEWorker-2 {}] master.SplitLogManager(171): hdfs://localhost:33505/user/jenkins/test-data/e37915a4-8313-e328-91fb-37d541e8b214/WALs/1ab588ae992e,40611,1732433713633-splitting dir is empty, no logs to split. 2024-11-24T07:35:30,913 INFO [PEWorker-2 {}] master.SplitWALManager(105): 1ab588ae992e,40611,1732433713633 WAL count=0, meta=true 2024-11-24T07:35:30,913 DEBUG [PEWorker-2 {}] procedure.ServerCrashProcedure(329): Check if 1ab588ae992e,40611,1732433713633 WAL splitting is done? wals=0, meta=true 2024-11-24T07:35:30,916 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=16, ppid=13, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-11-24T07:35:30,918 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=16, ppid=13, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-24T07:35:30,919 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(269): Starting pid=16, ppid=13, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-11-24T07:35:30,992 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-24T07:35:30,994 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=1ab588ae992e,40611,1732433713633, seqNum=-1] 2024-11-24T07:35:30,994 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] ipc.AbstractRpcClient(357): Not trying to connect to 1ab588ae992e:40611 this server is in the failed servers list 2024-11-24T07:35:30,994 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=1ab588ae992e,40611,1732433713633, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=1ab588ae992e,40611,1732433713633, seqNum=-1, error=org.apache.hadoop.hbase.ipc.FailedServerException: Call to address=1ab588ae992e:40611 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: 1ab588ae992e:40611 2024-11-24T07:35:30,994 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:meta,,1.1588230740, hostname=1ab588ae992e,40611,1732433713633, seqNum=-1 is org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: 1ab588ae992e:40611 2024-11-24T07:35:30,994 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:meta,,1.1588230740, hostname=1ab588ae992e,40611,1732433713633, seqNum=-1 from cache 2024-11-24T07:35:31,069 DEBUG [1ab588ae992e:37719 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=2, allServersCount=2 2024-11-24T07:35:31,070 DEBUG [1ab588ae992e:37719 {}] balancer.BalancerClusterState(204): Hosts are {1ab588ae992e=0} racks are {/default-rack=0} 2024-11-24T07:35:31,070 DEBUG [1ab588ae992e:37719 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-11-24T07:35:31,070 DEBUG [1ab588ae992e:37719 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-11-24T07:35:31,070 DEBUG [1ab588ae992e:37719 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-11-24T07:35:31,070 DEBUG [1ab588ae992e:37719 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-11-24T07:35:31,070 INFO [1ab588ae992e:37719 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-11-24T07:35:31,070 INFO [1ab588ae992e:37719 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-11-24T07:35:31,070 DEBUG [1ab588ae992e:37719 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-11-24T07:35:31,071 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=16 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=1ab588ae992e,40419,1732433713579 2024-11-24T07:35:31,072 INFO [PEWorker-1 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 1ab588ae992e,40419,1732433713579, state=OPENING 2024-11-24T07:35:31,161 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37719-0x1016bc48eea0000, quorum=127.0.0.1:49244, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-24T07:35:31,161 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46433-0x1016bc48eea0001, quorum=127.0.0.1:49244, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-24T07:35:31,161 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40419-0x1016bc48eea0002, quorum=127.0.0.1:49244, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-24T07:35:31,161 DEBUG [PEWorker-1 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=16, ppid=13, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-24T07:35:31,161 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-24T07:35:31,161 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-24T07:35:31,161 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-24T07:35:31,161 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=17, ppid=16, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=1ab588ae992e,40419,1732433713579}] 2024-11-24T07:35:31,314 DEBUG [RSProcedureDispatcher-pool-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-24T07:35:31,318 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-4-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:58757, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-24T07:35:31,325 INFO [RS_OPEN_META-regionserver/1ab588ae992e:0-0 {event_type=M_RS_OPEN_META, pid=17}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-11-24T07:35:31,325 INFO [RS_OPEN_META-regionserver/1ab588ae992e:0-0 {event_type=M_RS_OPEN_META, pid=17}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-24T07:35:31,325 INFO [RS_OPEN_META-regionserver/1ab588ae992e:0-0 {event_type=M_RS_OPEN_META, pid=17}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-11-24T07:35:31,327 INFO [RS_OPEN_META-regionserver/1ab588ae992e:0-0 {event_type=M_RS_OPEN_META, pid=17}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=1ab588ae992e%2C40419%2C1732433713579.meta, suffix=.meta, logDir=hdfs://localhost:33505/user/jenkins/test-data/e37915a4-8313-e328-91fb-37d541e8b214/WALs/1ab588ae992e,40419,1732433713579, archiveDir=hdfs://localhost:33505/user/jenkins/test-data/e37915a4-8313-e328-91fb-37d541e8b214/oldWALs, maxLogs=32 2024-11-24T07:35:31,341 DEBUG [RS_OPEN_META-regionserver/1ab588ae992e:0-0 {event_type=M_RS_OPEN_META, pid=17}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/e37915a4-8313-e328-91fb-37d541e8b214/WALs/1ab588ae992e,40419,1732433713579/1ab588ae992e%2C40419%2C1732433713579.meta.1732433731328.meta, exclude list is [], retry=0 2024-11-24T07:35:31,343 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41971,DS-a67a12dc-5f72-4d11-ac5a-3e225e33c62a,DISK] 2024-11-24T07:35:31,343 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41161,DS-b0260eac-1380-4e84-bf22-a60109fdf5f6,DISK] 2024-11-24T07:35:31,343 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:42685,DS-467ee9a1-f3ca-4279-9b61-69a2a6a0308f,DISK] 2024-11-24T07:35:31,345 INFO [RS_OPEN_META-regionserver/1ab588ae992e:0-0 {event_type=M_RS_OPEN_META, pid=17}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/e37915a4-8313-e328-91fb-37d541e8b214/WALs/1ab588ae992e,40419,1732433713579/1ab588ae992e%2C40419%2C1732433713579.meta.1732433731328.meta 2024-11-24T07:35:31,346 DEBUG [RS_OPEN_META-regionserver/1ab588ae992e:0-0 {event_type=M_RS_OPEN_META, pid=17}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:37111:37111),(127.0.0.1/127.0.0.1:40647:40647),(127.0.0.1/127.0.0.1:35537:35537)] 2024-11-24T07:35:31,346 DEBUG [RS_OPEN_META-regionserver/1ab588ae992e:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-11-24T07:35:31,346 DEBUG [RS_OPEN_META-regionserver/1ab588ae992e:0-0 {event_type=M_RS_OPEN_META, pid=17}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-11-24T07:35:31,346 DEBUG [RS_OPEN_META-regionserver/1ab588ae992e:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-11-24T07:35:31,346 INFO [RS_OPEN_META-regionserver/1ab588ae992e:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-11-24T07:35:31,347 DEBUG [RS_OPEN_META-regionserver/1ab588ae992e:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-11-24T07:35:31,347 DEBUG [RS_OPEN_META-regionserver/1ab588ae992e:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T07:35:31,347 DEBUG [RS_OPEN_META-regionserver/1ab588ae992e:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-11-24T07:35:31,347 DEBUG [RS_OPEN_META-regionserver/1ab588ae992e:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-11-24T07:35:31,348 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-24T07:35:31,349 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-24T07:35:31,349 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T07:35:31,350 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-24T07:35:31,350 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-24T07:35:31,350 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-24T07:35:31,350 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T07:35:31,351 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-24T07:35:31,351 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-24T07:35:31,352 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-24T07:35:31,352 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T07:35:31,352 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-24T07:35:31,352 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-24T07:35:31,353 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-24T07:35:31,353 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T07:35:31,353 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-24T07:35:31,353 DEBUG [RS_OPEN_META-regionserver/1ab588ae992e:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-24T07:35:31,354 DEBUG [RS_OPEN_META-regionserver/1ab588ae992e:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33505/user/jenkins/test-data/e37915a4-8313-e328-91fb-37d541e8b214/data/hbase/meta/1588230740 2024-11-24T07:35:31,356 DEBUG [RS_OPEN_META-regionserver/1ab588ae992e:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:33505/user/jenkins/test-data/e37915a4-8313-e328-91fb-37d541e8b214/data/hbase/meta/1588230740 2024-11-24T07:35:31,356 INFO [RS_OPEN_META-regionserver/1ab588ae992e:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:33505/user/jenkins/test-data/e37915a4-8313-e328-91fb-37d541e8b214/data/hbase/meta/1588230740/recovered.edits/0000000000000000018 2024-11-24T07:35:31,358 DEBUG [RS_OPEN_META-regionserver/1ab588ae992e:0-0 {event_type=M_RS_OPEN_META, pid=17}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:33505/user/jenkins/test-data/e37915a4-8313-e328-91fb-37d541e8b214/data/hbase/meta/1588230740/recovered.edits/0000000000000000018: isRecoveredEdits=true, hasTagCompression=true, hasValueCompression=true, valueCompressionType=GZ 2024-11-24T07:35:31,361 DEBUG [RS_OPEN_META-regionserver/1ab588ae992e:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegion(5793): Applied 40, skipped 0, firstSequenceIdInLog=4, maxSequenceIdInLog=18, path=hdfs://localhost:33505/user/jenkins/test-data/e37915a4-8313-e328-91fb-37d541e8b214/data/hbase/meta/1588230740/recovered.edits/0000000000000000018 2024-11-24T07:35:31,362 INFO [RS_OPEN_META-regionserver/1ab588ae992e:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=5.67 KB heapSize=9.66 KB 2024-11-24T07:35:31,382 DEBUG [RS_OPEN_META-regionserver/1ab588ae992e:0-0 {event_type=M_RS_OPEN_META, pid=17}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33505/user/jenkins/test-data/e37915a4-8313-e328-91fb-37d541e8b214/data/hbase/meta/1588230740/.tmp/info/41cfa193974c4f8d8d4f837050ff64e3 is 205, key is testReplayEditsAfterRegionMovedWithMultiCF,,1732433728060.567d9cab2ff71509e601db9c0fa44478./info:regioninfo/1732433730286/Put/seqid=0 2024-11-24T07:35:31,388 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41161 is added to blk_1073741894_1072 (size=11177) 2024-11-24T07:35:31,389 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41971 is added to blk_1073741894_1072 (size=11177) 2024-11-24T07:35:31,389 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42685 is added to blk_1073741894_1072 (size=11177) 2024-11-24T07:35:31,389 INFO [RS_OPEN_META-regionserver/1ab588ae992e:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=5.46 KB at sequenceid=18 (bloomFilter=true), to=hdfs://localhost:33505/user/jenkins/test-data/e37915a4-8313-e328-91fb-37d541e8b214/data/hbase/meta/1588230740/.tmp/info/41cfa193974c4f8d8d4f837050ff64e3 2024-11-24T07:35:31,411 DEBUG [RS_OPEN_META-regionserver/1ab588ae992e:0-0 {event_type=M_RS_OPEN_META, pid=17}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33505/user/jenkins/test-data/e37915a4-8313-e328-91fb-37d541e8b214/data/hbase/meta/1588230740/.tmp/ns/e2e446d0a557482392ec6a72c9f20311 is 43, key is default/ns:d/1732433715947/Put/seqid=0 2024-11-24T07:35:31,417 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41971 is added to blk_1073741895_1073 (size=5153) 2024-11-24T07:35:31,418 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41161 is added to blk_1073741895_1073 (size=5153) 2024-11-24T07:35:31,418 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42685 is added to blk_1073741895_1073 (size=5153) 2024-11-24T07:35:31,418 INFO [RS_OPEN_META-regionserver/1ab588ae992e:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=74 B at sequenceid=18 (bloomFilter=true), to=hdfs://localhost:33505/user/jenkins/test-data/e37915a4-8313-e328-91fb-37d541e8b214/data/hbase/meta/1588230740/.tmp/ns/e2e446d0a557482392ec6a72c9f20311 2024-11-24T07:35:31,436 DEBUG [RS_OPEN_META-regionserver/1ab588ae992e:0-0 {event_type=M_RS_OPEN_META, pid=17}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33505/user/jenkins/test-data/e37915a4-8313-e328-91fb-37d541e8b214/data/hbase/meta/1588230740/.tmp/table/80bc77759545446fbdf00d99ce2a196c is 78, key is testReplayEditsAfterRegionMovedWithMultiCF/table:state/1732433728483/Put/seqid=0 2024-11-24T07:35:31,442 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41161 is added to blk_1073741896_1074 (size=5431) 2024-11-24T07:35:31,443 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42685 is added to blk_1073741896_1074 (size=5431) 2024-11-24T07:35:31,443 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41971 is added to blk_1073741896_1074 (size=5431) 2024-11-24T07:35:31,443 INFO [RS_OPEN_META-regionserver/1ab588ae992e:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=148 B at sequenceid=18 (bloomFilter=true), to=hdfs://localhost:33505/user/jenkins/test-data/e37915a4-8313-e328-91fb-37d541e8b214/data/hbase/meta/1588230740/.tmp/table/80bc77759545446fbdf00d99ce2a196c 2024-11-24T07:35:31,449 DEBUG [RS_OPEN_META-regionserver/1ab588ae992e:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33505/user/jenkins/test-data/e37915a4-8313-e328-91fb-37d541e8b214/data/hbase/meta/1588230740/.tmp/info/41cfa193974c4f8d8d4f837050ff64e3 as hdfs://localhost:33505/user/jenkins/test-data/e37915a4-8313-e328-91fb-37d541e8b214/data/hbase/meta/1588230740/info/41cfa193974c4f8d8d4f837050ff64e3 2024-11-24T07:35:31,454 INFO [RS_OPEN_META-regionserver/1ab588ae992e:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33505/user/jenkins/test-data/e37915a4-8313-e328-91fb-37d541e8b214/data/hbase/meta/1588230740/info/41cfa193974c4f8d8d4f837050ff64e3, entries=36, sequenceid=18, filesize=10.9 K 2024-11-24T07:35:31,455 DEBUG [RS_OPEN_META-regionserver/1ab588ae992e:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33505/user/jenkins/test-data/e37915a4-8313-e328-91fb-37d541e8b214/data/hbase/meta/1588230740/.tmp/ns/e2e446d0a557482392ec6a72c9f20311 as hdfs://localhost:33505/user/jenkins/test-data/e37915a4-8313-e328-91fb-37d541e8b214/data/hbase/meta/1588230740/ns/e2e446d0a557482392ec6a72c9f20311 2024-11-24T07:35:31,461 INFO [RS_OPEN_META-regionserver/1ab588ae992e:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33505/user/jenkins/test-data/e37915a4-8313-e328-91fb-37d541e8b214/data/hbase/meta/1588230740/ns/e2e446d0a557482392ec6a72c9f20311, entries=2, sequenceid=18, filesize=5.0 K 2024-11-24T07:35:31,462 DEBUG [RS_OPEN_META-regionserver/1ab588ae992e:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33505/user/jenkins/test-data/e37915a4-8313-e328-91fb-37d541e8b214/data/hbase/meta/1588230740/.tmp/table/80bc77759545446fbdf00d99ce2a196c as hdfs://localhost:33505/user/jenkins/test-data/e37915a4-8313-e328-91fb-37d541e8b214/data/hbase/meta/1588230740/table/80bc77759545446fbdf00d99ce2a196c 2024-11-24T07:35:31,468 INFO [RS_OPEN_META-regionserver/1ab588ae992e:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33505/user/jenkins/test-data/e37915a4-8313-e328-91fb-37d541e8b214/data/hbase/meta/1588230740/table/80bc77759545446fbdf00d99ce2a196c, entries=2, sequenceid=18, filesize=5.3 K 2024-11-24T07:35:31,469 INFO [RS_OPEN_META-regionserver/1ab588ae992e:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegion(3140): Finished flush of dataSize ~5.67 KB/5811, heapSize ~9.37 KB/9592, currentSize=0 B/0 for 1588230740 in 108ms, sequenceid=18, compaction requested=false; wal=null 2024-11-24T07:35:31,470 DEBUG [RS_OPEN_META-regionserver/1ab588ae992e:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:33505/user/jenkins/test-data/e37915a4-8313-e328-91fb-37d541e8b214/data/hbase/meta/1588230740/recovered.edits/0000000000000000018 2024-11-24T07:35:31,471 DEBUG [RS_OPEN_META-regionserver/1ab588ae992e:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-24T07:35:31,471 DEBUG [RS_OPEN_META-regionserver/1ab588ae992e:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-24T07:35:31,472 DEBUG [RS_OPEN_META-regionserver/1ab588ae992e:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-24T07:35:31,473 DEBUG [RS_OPEN_META-regionserver/1ab588ae992e:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-24T07:35:31,475 DEBUG [RS_OPEN_META-regionserver/1ab588ae992e:0-0 {event_type=M_RS_OPEN_META, pid=17}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:33505/user/jenkins/test-data/e37915a4-8313-e328-91fb-37d541e8b214/data/hbase/meta/1588230740/recovered.edits/18.seqid, newMaxSeqId=18, maxSeqId=1 2024-11-24T07:35:31,476 INFO [RS_OPEN_META-regionserver/1ab588ae992e:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=19; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=59377470, jitterRate=-0.1152067482471466}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-24T07:35:31,477 DEBUG [RS_OPEN_META-regionserver/1ab588ae992e:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-11-24T07:35:31,477 DEBUG [RS_OPEN_META-regionserver/1ab588ae992e:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1732433731347Writing region info on filesystem at 1732433731347Initializing all the Stores at 1732433731348 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732433731348Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732433731348Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732433731348Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732433731348Obtaining lock to block concurrent updates at 1732433731362 (+14 ms)Preparing flush snapshotting stores in 1588230740 at 1732433731362Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=5811, getHeapSize=9832, getOffHeapSize=0, getCellsCount=40 at 1732433731362Flushing stores of hbase:meta,,1.1588230740 at 1732433731362Flushing 1588230740/info: creating writer at 1732433731362Flushing 1588230740/info: appending metadata at 1732433731382 (+20 ms)Flushing 1588230740/info: closing flushed file at 1732433731382Flushing 1588230740/ns: creating writer at 1732433731395 (+13 ms)Flushing 1588230740/ns: appending metadata at 1732433731411 (+16 ms)Flushing 1588230740/ns: closing flushed file at 1732433731411Flushing 1588230740/table: creating writer at 1732433731423 (+12 ms)Flushing 1588230740/table: appending metadata at 1732433731436 (+13 ms)Flushing 1588230740/table: closing flushed file at 1732433731436Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@23411058: reopening flushed file at 1732433731448 (+12 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@4ce42c53: reopening flushed file at 1732433731454 (+6 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@3a5cd957: reopening flushed file at 1732433731462 (+8 ms)Finished flush of dataSize ~5.67 KB/5811, heapSize ~9.37 KB/9592, currentSize=0 B/0 for 1588230740 in 108ms, sequenceid=18, compaction requested=false; wal=null at 1732433731469 (+7 ms)Cleaning up temporary data from old regions at 1732433731471 (+2 ms)Running coprocessor post-open hooks at 1732433731477 (+6 ms)Region opened successfully at 1732433731477 2024-11-24T07:35:31,478 INFO [RS_OPEN_META-regionserver/1ab588ae992e:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=17, masterSystemTime=1732433731314 2024-11-24T07:35:31,481 DEBUG [RS_OPEN_META-regionserver/1ab588ae992e:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-11-24T07:35:31,481 INFO [RS_OPEN_META-regionserver/1ab588ae992e:0-0 {event_type=M_RS_OPEN_META, pid=17}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-11-24T07:35:31,481 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=16 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=19, regionLocation=1ab588ae992e,40419,1732433713579 2024-11-24T07:35:31,483 INFO [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 1ab588ae992e,40419,1732433713579, state=OPEN 2024-11-24T07:35:31,502 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-24T07:35:31,503 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=1ab588ae992e,40419,1732433713579, seqNum=-1] 2024-11-24T07:35:31,504 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-24T07:35:31,506 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-4-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:58196, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-24T07:35:31,508 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40419-0x1016bc48eea0002, quorum=127.0.0.1:49244, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-24T07:35:31,508 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46433-0x1016bc48eea0001, quorum=127.0.0.1:49244, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-24T07:35:31,508 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37719-0x1016bc48eea0000, quorum=127.0.0.1:49244, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-24T07:35:31,508 DEBUG [PEWorker-5 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=17, ppid=16, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=1ab588ae992e,40419,1732433713579 2024-11-24T07:35:31,508 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-24T07:35:31,508 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-24T07:35:31,508 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-24T07:35:31,514 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=17, resume processing ppid=16 2024-11-24T07:35:31,515 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=17, ppid=16, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=1ab588ae992e,40419,1732433713579 in 347 msec 2024-11-24T07:35:31,518 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=16, resume processing ppid=13 2024-11-24T07:35:31,518 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=16, ppid=13, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 599 msec 2024-11-24T07:35:31,518 INFO [PEWorker-3 {}] procedure.ServerCrashProcedure(207): 1ab588ae992e,40611,1732433713633 had 2 regions 2024-11-24T07:35:31,519 INFO [PEWorker-3 {}] procedure.ServerCrashProcedure(339): Splitting WALs pid=13, state=RUNNABLE:SERVER_CRASH_SPLIT_LOGS, hasLock=true; ServerCrashProcedure 1ab588ae992e,40611,1732433713633, splitWal=true, meta=true, isMeta: false 2024-11-24T07:35:31,521 INFO [PEWorker-3 {}] master.SplitWALManager(105): 1ab588ae992e,40611,1732433713633 WAL count=1, meta=false 2024-11-24T07:35:31,521 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=18, ppid=13, state=RUNNABLE:ACQUIRE_SPLIT_WAL_WORKER, hasLock=false; SplitWALProcedure 1ab588ae992e%2C40611%2C1732433713633.1732433715254}] 2024-11-24T07:35:31,524 DEBUG [PEWorker-1 {}] master.SplitWALManager(158): Acquired split WAL worker=1ab588ae992e,46433,1732433713473 2024-11-24T07:35:31,525 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=19, ppid=18, state=RUNNABLE, hasLock=false; SplitWALRemoteProcedure 1ab588ae992e%2C40611%2C1732433713633.1732433715254, worker=1ab588ae992e,46433,1732433713473}] 2024-11-24T07:35:31,622 DEBUG [Async-Client-Retry-Timer-pool-0 {}] ipc.AbstractRpcClient(357): Not trying to connect to 1ab588ae992e:40611 this server is in the failed servers list 2024-11-24T07:35:31,622 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.AsyncRegionLocatorHelper(64): Try updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1732433728060.567d9cab2ff71509e601db9c0fa44478., hostname=1ab588ae992e,40611,1732433713633, seqNum=12 , the old value is region=testReplayEditsAfterRegionMovedWithMultiCF,,1732433728060.567d9cab2ff71509e601db9c0fa44478., hostname=1ab588ae992e,40611,1732433713633, seqNum=12, error=org.apache.hadoop.hbase.ipc.FailedServerException: Call to address=1ab588ae992e:40611 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: 1ab588ae992e:40611 2024-11-24T07:35:31,622 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1732433728060.567d9cab2ff71509e601db9c0fa44478., hostname=1ab588ae992e,40611,1732433713633, seqNum=12 is org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: 1ab588ae992e:40611 2024-11-24T07:35:31,622 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.AsyncRegionLocatorHelper(88): Try removing region=testReplayEditsAfterRegionMovedWithMultiCF,,1732433728060.567d9cab2ff71509e601db9c0fa44478., hostname=1ab588ae992e,40611,1732433713633, seqNum=12 from cache 2024-11-24T07:35:31,678 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46433 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SplitWALCallable, pid=19 2024-11-24T07:35:31,693 INFO [RS_LOG_REPLAY_OPS-regionserver/1ab588ae992e:0-1 {event_type=RS_LOG_REPLAY, pid=19}] wal.WALSplitter(299): Splitting hdfs://localhost:33505/user/jenkins/test-data/e37915a4-8313-e328-91fb-37d541e8b214/WALs/1ab588ae992e,40611,1732433713633-splitting/1ab588ae992e%2C40611%2C1732433713633.1732433715254, size=1.4 K (1405bytes) 2024-11-24T07:35:31,693 INFO [RS_LOG_REPLAY_OPS-regionserver/1ab588ae992e:0-1 {event_type=RS_LOG_REPLAY, pid=19}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:33505/user/jenkins/test-data/e37915a4-8313-e328-91fb-37d541e8b214/WALs/1ab588ae992e,40611,1732433713633-splitting/1ab588ae992e%2C40611%2C1732433713633.1732433715254 2024-11-24T07:35:31,694 INFO [RS_LOG_REPLAY_OPS-regionserver/1ab588ae992e:0-1 {event_type=RS_LOG_REPLAY, pid=19}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:33505/user/jenkins/test-data/e37915a4-8313-e328-91fb-37d541e8b214/WALs/1ab588ae992e,40611,1732433713633-splitting/1ab588ae992e%2C40611%2C1732433713633.1732433715254 after 1ms 2024-11-24T07:35:31,696 DEBUG [RS_LOG_REPLAY_OPS-regionserver/1ab588ae992e:0-1 {event_type=RS_LOG_REPLAY, pid=19}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:33505/user/jenkins/test-data/e37915a4-8313-e328-91fb-37d541e8b214/WALs/1ab588ae992e,40611,1732433713633-splitting/1ab588ae992e%2C40611%2C1732433713633.1732433715254: isRecoveredEdits=false, hasTagCompression=true, hasValueCompression=true, valueCompressionType=GZ 2024-11-24T07:35:31,697 INFO [RS_LOG_REPLAY_OPS-regionserver/1ab588ae992e:0-1 {event_type=RS_LOG_REPLAY, pid=19}] wal.WALSplitter(310): Open hdfs://localhost:33505/user/jenkins/test-data/e37915a4-8313-e328-91fb-37d541e8b214/WALs/1ab588ae992e,40611,1732433713633-splitting/1ab588ae992e%2C40611%2C1732433713633.1732433715254 took 4ms 2024-11-24T07:35:31,700 DEBUG [RS_LOG_REPLAY_OPS-regionserver/1ab588ae992e:0-1 {event_type=RS_LOG_REPLAY, pid=19}] wal.WALSplitter(352): Last flushed sequenceid for 567d9cab2ff71509e601db9c0fa44478: last_flushed_sequence_id: 12 store_sequence_id { family_name: "cf1" sequence_id: 12 } store_sequence_id { family_name: "cf2" sequence_id: 12 } 2024-11-24T07:35:31,700 DEBUG [RS_LOG_REPLAY_OPS-regionserver/1ab588ae992e:0-1 {event_type=RS_LOG_REPLAY, pid=19}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:33505/user/jenkins/test-data/e37915a4-8313-e328-91fb-37d541e8b214/WALs/1ab588ae992e,40611,1732433713633-splitting/1ab588ae992e%2C40611%2C1732433713633.1732433715254 so closing down 2024-11-24T07:35:31,700 DEBUG [RS_LOG_REPLAY_OPS-regionserver/1ab588ae992e:0-1 {event_type=RS_LOG_REPLAY, pid=19}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-24T07:35:31,700 INFO [RS_LOG_REPLAY_OPS-regionserver/1ab588ae992e:0-1 {event_type=RS_LOG_REPLAY, pid=19}] wal.OutputSink(145): 3 split writer threads finished 2024-11-24T07:35:31,700 INFO [RS_LOG_REPLAY_OPS-regionserver/1ab588ae992e:0-1 {event_type=RS_LOG_REPLAY, pid=19}] wal.WALSplitter(425): Processed 6 edits across 0 Regions in 3 ms; skipped=6; WAL=hdfs://localhost:33505/user/jenkins/test-data/e37915a4-8313-e328-91fb-37d541e8b214/WALs/1ab588ae992e,40611,1732433713633-splitting/1ab588ae992e%2C40611%2C1732433713633.1732433715254, size=1.4 K, length=1405, corrupted=false, cancelled=false 2024-11-24T07:35:31,700 DEBUG [RS_LOG_REPLAY_OPS-regionserver/1ab588ae992e:0-1 {event_type=RS_LOG_REPLAY, pid=19}] wal.WALSplitter(428): Completed split of hdfs://localhost:33505/user/jenkins/test-data/e37915a4-8313-e328-91fb-37d541e8b214/WALs/1ab588ae992e,40611,1732433713633-splitting/1ab588ae992e%2C40611%2C1732433713633.1732433715254, journal: Splitting hdfs://localhost:33505/user/jenkins/test-data/e37915a4-8313-e328-91fb-37d541e8b214/WALs/1ab588ae992e,40611,1732433713633-splitting/1ab588ae992e%2C40611%2C1732433713633.1732433715254, size=1.4 K (1405bytes) at 1732433731693Finishing writing output for hdfs://localhost:33505/user/jenkins/test-data/e37915a4-8313-e328-91fb-37d541e8b214/WALs/1ab588ae992e,40611,1732433713633-splitting/1ab588ae992e%2C40611%2C1732433713633.1732433715254 so closing down at 1732433731700 (+7 ms)3 split writer threads finished at 1732433731700Processed 6 edits across 0 Regions in 3 ms; skipped=6; WAL=hdfs://localhost:33505/user/jenkins/test-data/e37915a4-8313-e328-91fb-37d541e8b214/WALs/1ab588ae992e,40611,1732433713633-splitting/1ab588ae992e%2C40611%2C1732433713633.1732433715254, size=1.4 K, length=1405, corrupted=false, cancelled=false at 1732433731700 2024-11-24T07:35:31,700 DEBUG [RS_LOG_REPLAY_OPS-regionserver/1ab588ae992e:0-1 {event_type=RS_LOG_REPLAY, pid=19}] regionserver.SplitLogWorker(218): Done splitting WAL hdfs://localhost:33505/user/jenkins/test-data/e37915a4-8313-e328-91fb-37d541e8b214/WALs/1ab588ae992e,40611,1732433713633-splitting/1ab588ae992e%2C40611%2C1732433713633.1732433715254 2024-11-24T07:35:31,701 DEBUG [RS_LOG_REPLAY_OPS-regionserver/1ab588ae992e:0-1 {event_type=RS_LOG_REPLAY, pid=19}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=19 2024-11-24T07:35:31,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37719 {}] master.HMaster(4169): Remote procedure done, pid=19 2024-11-24T07:35:31,705 INFO [PEWorker-5 {}] wal.WALSplitUtil(143): Moved hdfs://localhost:33505/user/jenkins/test-data/e37915a4-8313-e328-91fb-37d541e8b214/WALs/1ab588ae992e,40611,1732433713633-splitting/1ab588ae992e%2C40611%2C1732433713633.1732433715254 to hdfs://localhost:33505/user/jenkins/test-data/e37915a4-8313-e328-91fb-37d541e8b214/oldWALs 2024-11-24T07:35:31,708 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=19, resume processing ppid=18 2024-11-24T07:35:31,708 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=19, ppid=18, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 1ab588ae992e%2C40611%2C1732433713633.1732433715254, worker=1ab588ae992e,46433,1732433713473 in 180 msec 2024-11-24T07:35:31,710 DEBUG [PEWorker-2 {}] master.SplitWALManager(172): Release split WAL worker=1ab588ae992e,46433,1732433713473 2024-11-24T07:35:31,712 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=18, resume processing ppid=13 2024-11-24T07:35:31,712 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=18, ppid=13, state=SUCCESS, hasLock=false; SplitWALProcedure 1ab588ae992e%2C40611%2C1732433713633.1732433715254, worker=1ab588ae992e,46433,1732433713473 in 189 msec 2024-11-24T07:35:31,714 INFO [PEWorker-3 {}] master.SplitLogManager(171): hdfs://localhost:33505/user/jenkins/test-data/e37915a4-8313-e328-91fb-37d541e8b214/WALs/1ab588ae992e,40611,1732433713633-splitting dir is empty, no logs to split. 2024-11-24T07:35:31,714 INFO [PEWorker-3 {}] master.SplitWALManager(105): 1ab588ae992e,40611,1732433713633 WAL count=0, meta=false 2024-11-24T07:35:31,714 DEBUG [PEWorker-3 {}] procedure.ServerCrashProcedure(329): Check if 1ab588ae992e,40611,1732433713633 WAL splitting is done? wals=0, meta=false 2024-11-24T07:35:31,717 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=20, ppid=13, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=567d9cab2ff71509e601db9c0fa44478, ASSIGN}] 2024-11-24T07:35:31,719 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=20, ppid=13, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=567d9cab2ff71509e601db9c0fa44478, ASSIGN 2024-11-24T07:35:31,720 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=20, ppid=13, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=567d9cab2ff71509e601db9c0fa44478, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-11-24T07:35:31,839 DEBUG [RPCClient-NioEventLoopGroup-6-8 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testReplayEditsAfterRegionMovedWithMultiCF', row='r1', locateType=CURRENT is [region=testReplayEditsAfterRegionMovedWithMultiCF,,1732433728060.567d9cab2ff71509e601db9c0fa44478., hostname=1ab588ae992e,40611,1732433713633, seqNum=18] 2024-11-24T07:35:31,839 DEBUG [RPCClient-NioEventLoopGroup-6-8 {}] ipc.AbstractRpcClient(357): Not trying to connect to 1ab588ae992e:40611 this server is in the failed servers list 2024-11-24T07:35:31,839 DEBUG [RPCClient-NioEventLoopGroup-6-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1732433728060.567d9cab2ff71509e601db9c0fa44478., hostname=1ab588ae992e,40611,1732433713633, seqNum=18 , the old value is region=testReplayEditsAfterRegionMovedWithMultiCF,,1732433728060.567d9cab2ff71509e601db9c0fa44478., hostname=1ab588ae992e,40611,1732433713633, seqNum=18, error=org.apache.hadoop.hbase.ipc.FailedServerException: Call to address=1ab588ae992e:40611 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: 1ab588ae992e:40611 2024-11-24T07:35:31,840 DEBUG [RPCClient-NioEventLoopGroup-6-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1732433728060.567d9cab2ff71509e601db9c0fa44478., hostname=1ab588ae992e,40611,1732433713633, seqNum=18 is org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: 1ab588ae992e:40611 2024-11-24T07:35:31,840 DEBUG [RPCClient-NioEventLoopGroup-6-8 {}] client.AsyncRegionLocatorHelper(88): Try removing region=testReplayEditsAfterRegionMovedWithMultiCF,,1732433728060.567d9cab2ff71509e601db9c0fa44478., hostname=1ab588ae992e,40611,1732433713633, seqNum=18 from cache 2024-11-24T07:35:31,870 DEBUG [1ab588ae992e:37719 {}] balancer.BalancerClusterState(204): Hosts are {1ab588ae992e=0} racks are {/default-rack=0} 2024-11-24T07:35:31,871 DEBUG [1ab588ae992e:37719 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-11-24T07:35:31,871 DEBUG [1ab588ae992e:37719 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-11-24T07:35:31,871 DEBUG [1ab588ae992e:37719 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-11-24T07:35:31,871 DEBUG [1ab588ae992e:37719 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-11-24T07:35:31,871 INFO [1ab588ae992e:37719 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-11-24T07:35:31,871 INFO [1ab588ae992e:37719 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-11-24T07:35:31,871 DEBUG [1ab588ae992e:37719 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-11-24T07:35:31,871 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=20 updating hbase:meta row=567d9cab2ff71509e601db9c0fa44478, regionState=OPENING, regionLocation=1ab588ae992e,40419,1732433713579 2024-11-24T07:35:31,872 WARN [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] ipc.NettyRpcConnection$2(409): Exception encountered while connecting to the server 1ab588ae992e:40611 org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 1ab588ae992e/172.17.0.3:40611 Caused by: java.net.ConnectException: finishConnect(..) failed: Connection refused at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.newConnectException0(Errors.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.handleConnectErrno(Errors.java:131) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Socket.finishConnect(Socket.java:359) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.doFinishConnect(AbstractEpollChannel.java:715) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:692) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:491) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T07:35:31,873 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=1ab588ae992e,40611,1732433713633, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=1ab588ae992e,40611,1732433713633, seqNum=-1, error=java.net.ConnectException: Call to address=1ab588ae992e:40611 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 1ab588ae992e/172.17.0.3:40611 2024-11-24T07:35:31,873 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:meta,,1.1588230740, hostname=1ab588ae992e,40611,1732433713633, seqNum=-1 is java.net.ConnectException: finishConnect(..) failed: Connection refused 2024-11-24T07:35:31,873 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:meta,,1.1588230740, hostname=1ab588ae992e,40611,1732433713633, seqNum=-1 from cache 2024-11-24T07:35:31,873 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] ipc.FailedServers(52): Added failed server with address 1ab588ae992e:40611 to list caused by org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 1ab588ae992e/172.17.0.3:40611 2024-11-24T07:35:31,982 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-24T07:35:31,982 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=1ab588ae992e,40419,1732433713579, seqNum=-1] 2024-11-24T07:35:31,982 DEBUG [Async-Client-Retry-Timer-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-24T07:35:31,984 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-4-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:49487, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-24T07:35:31,987 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=20, ppid=13, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=567d9cab2ff71509e601db9c0fa44478, ASSIGN because future has completed 2024-11-24T07:35:31,987 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=21, ppid=20, state=RUNNABLE, hasLock=false; OpenRegionProcedure 567d9cab2ff71509e601db9c0fa44478, server=1ab588ae992e,40419,1732433713579}] 2024-11-24T07:35:32,148 DEBUG [RPCClient-NioEventLoopGroup-6-8 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testReplayEditsAfterRegionMovedWithMultiCF', row='r1', locateType=CURRENT is [region=testReplayEditsAfterRegionMovedWithMultiCF,,1732433728060.567d9cab2ff71509e601db9c0fa44478., hostname=1ab588ae992e,40611,1732433713633, seqNum=18] 2024-11-24T07:35:32,148 DEBUG [RPCClient-NioEventLoopGroup-6-8 {}] ipc.AbstractRpcClient(357): Not trying to connect to 1ab588ae992e:40611 this server is in the failed servers list 2024-11-24T07:35:32,149 DEBUG [RPCClient-NioEventLoopGroup-6-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1732433728060.567d9cab2ff71509e601db9c0fa44478., hostname=1ab588ae992e,40611,1732433713633, seqNum=18 , the old value is region=testReplayEditsAfterRegionMovedWithMultiCF,,1732433728060.567d9cab2ff71509e601db9c0fa44478., hostname=1ab588ae992e,40611,1732433713633, seqNum=18, error=org.apache.hadoop.hbase.ipc.FailedServerException: Call to address=1ab588ae992e:40611 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: 1ab588ae992e:40611 2024-11-24T07:35:32,149 DEBUG [RPCClient-NioEventLoopGroup-6-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1732433728060.567d9cab2ff71509e601db9c0fa44478., hostname=1ab588ae992e,40611,1732433713633, seqNum=18 is org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: 1ab588ae992e:40611 2024-11-24T07:35:32,149 DEBUG [RPCClient-NioEventLoopGroup-6-8 {}] client.AsyncRegionLocatorHelper(88): Try removing region=testReplayEditsAfterRegionMovedWithMultiCF,,1732433728060.567d9cab2ff71509e601db9c0fa44478., hostname=1ab588ae992e,40611,1732433713633, seqNum=18 from cache 2024-11-24T07:35:32,177 INFO [RS_OPEN_REGION-regionserver/1ab588ae992e:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] handler.AssignRegionHandler(132): Open testReplayEditsAfterRegionMovedWithMultiCF,,1732433728060.567d9cab2ff71509e601db9c0fa44478. 2024-11-24T07:35:32,177 DEBUG [RS_OPEN_REGION-regionserver/1ab588ae992e:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(7752): Opening region: {ENCODED => 567d9cab2ff71509e601db9c0fa44478, NAME => 'testReplayEditsAfterRegionMovedWithMultiCF,,1732433728060.567d9cab2ff71509e601db9c0fa44478.', STARTKEY => '', ENDKEY => ''} 2024-11-24T07:35:32,177 DEBUG [RS_OPEN_REGION-regionserver/1ab588ae992e:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testReplayEditsAfterRegionMovedWithMultiCF 567d9cab2ff71509e601db9c0fa44478 2024-11-24T07:35:32,177 DEBUG [RS_OPEN_REGION-regionserver/1ab588ae992e:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(898): Instantiated testReplayEditsAfterRegionMovedWithMultiCF,,1732433728060.567d9cab2ff71509e601db9c0fa44478.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T07:35:32,177 DEBUG [RS_OPEN_REGION-regionserver/1ab588ae992e:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(7794): checking encryption for 567d9cab2ff71509e601db9c0fa44478 2024-11-24T07:35:32,177 DEBUG [RS_OPEN_REGION-regionserver/1ab588ae992e:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(7797): checking classloading for 567d9cab2ff71509e601db9c0fa44478 2024-11-24T07:35:32,179 INFO [StoreOpener-567d9cab2ff71509e601db9c0fa44478-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf1 of region 567d9cab2ff71509e601db9c0fa44478 2024-11-24T07:35:32,179 INFO [StoreOpener-567d9cab2ff71509e601db9c0fa44478-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 567d9cab2ff71509e601db9c0fa44478 columnFamilyName cf1 2024-11-24T07:35:32,179 DEBUG [StoreOpener-567d9cab2ff71509e601db9c0fa44478-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T07:35:32,185 DEBUG [StoreOpener-567d9cab2ff71509e601db9c0fa44478-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:33505/user/jenkins/test-data/e37915a4-8313-e328-91fb-37d541e8b214/data/default/testReplayEditsAfterRegionMovedWithMultiCF/567d9cab2ff71509e601db9c0fa44478/cf1/30e9069526ab4d65b8c0c6945d917fc5 2024-11-24T07:35:32,185 INFO [StoreOpener-567d9cab2ff71509e601db9c0fa44478-1 {}] regionserver.HStore(327): Store=567d9cab2ff71509e601db9c0fa44478/cf1, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T07:35:32,185 INFO [StoreOpener-567d9cab2ff71509e601db9c0fa44478-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf2 of region 567d9cab2ff71509e601db9c0fa44478 2024-11-24T07:35:32,186 INFO [StoreOpener-567d9cab2ff71509e601db9c0fa44478-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 567d9cab2ff71509e601db9c0fa44478 columnFamilyName cf2 2024-11-24T07:35:32,186 DEBUG [StoreOpener-567d9cab2ff71509e601db9c0fa44478-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T07:35:32,192 DEBUG [StoreOpener-567d9cab2ff71509e601db9c0fa44478-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:33505/user/jenkins/test-data/e37915a4-8313-e328-91fb-37d541e8b214/data/default/testReplayEditsAfterRegionMovedWithMultiCF/567d9cab2ff71509e601db9c0fa44478/cf2/476cd101fd09468e8d2393440995e674 2024-11-24T07:35:32,192 INFO [StoreOpener-567d9cab2ff71509e601db9c0fa44478-1 {}] regionserver.HStore(327): Store=567d9cab2ff71509e601db9c0fa44478/cf2, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T07:35:32,192 DEBUG [RS_OPEN_REGION-regionserver/1ab588ae992e:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(1038): replaying wal for 567d9cab2ff71509e601db9c0fa44478 2024-11-24T07:35:32,193 DEBUG [RS_OPEN_REGION-regionserver/1ab588ae992e:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33505/user/jenkins/test-data/e37915a4-8313-e328-91fb-37d541e8b214/data/default/testReplayEditsAfterRegionMovedWithMultiCF/567d9cab2ff71509e601db9c0fa44478 2024-11-24T07:35:32,194 DEBUG [RS_OPEN_REGION-regionserver/1ab588ae992e:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33505/user/jenkins/test-data/e37915a4-8313-e328-91fb-37d541e8b214/data/default/testReplayEditsAfterRegionMovedWithMultiCF/567d9cab2ff71509e601db9c0fa44478 2024-11-24T07:35:32,195 DEBUG [RS_OPEN_REGION-regionserver/1ab588ae992e:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(1048): stopping wal replay for 567d9cab2ff71509e601db9c0fa44478 2024-11-24T07:35:32,195 DEBUG [RS_OPEN_REGION-regionserver/1ab588ae992e:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(1060): Cleaning up temporary data for 567d9cab2ff71509e601db9c0fa44478 2024-11-24T07:35:32,195 DEBUG [RS_OPEN_REGION-regionserver/1ab588ae992e:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsAfterRegionMovedWithMultiCF descriptor;using region.getMemStoreFlushHeapSize/# of families (64.0 M)) instead. 2024-11-24T07:35:32,197 DEBUG [RS_OPEN_REGION-regionserver/1ab588ae992e:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(1093): writing seq id for 567d9cab2ff71509e601db9c0fa44478 2024-11-24T07:35:32,197 INFO [RS_OPEN_REGION-regionserver/1ab588ae992e:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(1114): Opened 567d9cab2ff71509e601db9c0fa44478; next sequenceid=18; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=74796428, jitterRate=0.11455363035202026}}}, FlushLargeStoresPolicy{flushSizeLowerBound=67108864} 2024-11-24T07:35:32,197 DEBUG [RS_OPEN_REGION-regionserver/1ab588ae992e:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 567d9cab2ff71509e601db9c0fa44478 2024-11-24T07:35:32,198 DEBUG [RS_OPEN_REGION-regionserver/1ab588ae992e:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(1006): Region open journal for 567d9cab2ff71509e601db9c0fa44478: Running coprocessor pre-open hook at 1732433732177Writing region info on filesystem at 1732433732177Initializing all the Stores at 1732433732178 (+1 ms)Instantiating store for column family {NAME => 'cf1', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732433732178Instantiating store for column family {NAME => 'cf2', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732433732178Cleaning up temporary data from old regions at 1732433732195 (+17 ms)Running coprocessor post-open hooks at 1732433732197 (+2 ms)Region opened successfully at 1732433732198 (+1 ms) 2024-11-24T07:35:32,199 INFO [RS_OPEN_REGION-regionserver/1ab588ae992e:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegionServer(2236): Post open deploy tasks for testReplayEditsAfterRegionMovedWithMultiCF,,1732433728060.567d9cab2ff71509e601db9c0fa44478., pid=21, masterSystemTime=1732433732173 2024-11-24T07:35:32,201 DEBUG [RS_OPEN_REGION-regionserver/1ab588ae992e:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegionServer(2266): Finished post open deploy task for testReplayEditsAfterRegionMovedWithMultiCF,,1732433728060.567d9cab2ff71509e601db9c0fa44478. 2024-11-24T07:35:32,201 INFO [RS_OPEN_REGION-regionserver/1ab588ae992e:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] handler.AssignRegionHandler(153): Opened testReplayEditsAfterRegionMovedWithMultiCF,,1732433728060.567d9cab2ff71509e601db9c0fa44478. 2024-11-24T07:35:32,202 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=20 updating hbase:meta row=567d9cab2ff71509e601db9c0fa44478, regionState=OPEN, openSeqNum=18, regionLocation=1ab588ae992e,40419,1732433713579 2024-11-24T07:35:32,204 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=21, ppid=20, state=RUNNABLE, hasLock=false; OpenRegionProcedure 567d9cab2ff71509e601db9c0fa44478, server=1ab588ae992e,40419,1732433713579 because future has completed 2024-11-24T07:35:32,207 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=21, resume processing ppid=20 2024-11-24T07:35:32,207 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=21, ppid=20, state=SUCCESS, hasLock=false; OpenRegionProcedure 567d9cab2ff71509e601db9c0fa44478, server=1ab588ae992e,40419,1732433713579 in 218 msec 2024-11-24T07:35:32,209 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=20, resume processing ppid=13 2024-11-24T07:35:32,209 INFO [PEWorker-2 {}] procedure.ServerCrashProcedure(291): removed crashed server 1ab588ae992e,40611,1732433713633 after splitting done 2024-11-24T07:35:32,209 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=20, ppid=13, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=567d9cab2ff71509e601db9c0fa44478, ASSIGN in 490 msec 2024-11-24T07:35:32,212 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=13, state=SUCCESS, hasLock=false; ServerCrashProcedure 1ab588ae992e,40611,1732433713633, splitWal=true, meta=true in 1.5420 sec 2024-11-24T07:35:32,669 DEBUG [RPCClient-NioEventLoopGroup-6-8 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testReplayEditsAfterRegionMovedWithMultiCF', row='r1', locateType=CURRENT is [region=testReplayEditsAfterRegionMovedWithMultiCF,,1732433728060.567d9cab2ff71509e601db9c0fa44478., hostname=1ab588ae992e,40419,1732433713579, seqNum=18] 2024-11-24T07:35:32,684 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestAsyncWALReplayValueCompression#testReplayEditsAfterRegionMovedWithMultiCF Thread=413 (was 412) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-4 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-1337235608-172.17.0.3-1732433708251:blk_1073741893_1071, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-7 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1052040828_22 at /127.0.0.1:52922 [Waiting for operation #34] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RSProcedureDispatcher-pool-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-3-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/089ee269-8889-d2f6-f8a2-4bb8eac365fc/cluster_a7d76481-495d-c9f6-361f-c7016573445a/data/data4 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1052040828_22 at /127.0.0.1:51590 [Waiting for operation #17] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-6-8 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:62) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:883) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-6-5 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:62) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:883) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-5 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1075778083_22 at /127.0.0.1:51640 [Receiving block BP-1337235608-172.17.0.3-1732433708251:blk_1073741893_1071] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-4-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1075778083_22 at /127.0.0.1:53060 [Receiving block BP-1337235608-172.17.0.3-1732433708251:blk_1073741893_1071] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1075778083_22 at /127.0.0.1:60130 [Waiting for operation #39] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1075778083_22 at /127.0.0.1:60216 [Receiving block BP-1337235608-172.17.0.3-1732433708251:blk_1073741893_1071] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-6 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_OPEN_REGION-regionserver/1ab588ae992e:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Abort regionserver monitor java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.util.TimerThread.mainLoop(Timer.java:563) java.base@17.0.11/java.util.TimerThread.run(Timer.java:516) Potentially hanging thread: AsyncFSWAL-0-hdfs://localhost:33505/user/jenkins/test-data/e37915a4-8313-e328-91fb-37d541e8b214-prefix:1ab588ae992e,40419,1732433713579.meta java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-1337235608-172.17.0.3-1732433708251:blk_1073741893_1071, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_LOG_REPLAY_OPS-regionserver/1ab588ae992e:0-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-3-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-1337235608-172.17.0.3-1732433708251:blk_1073741893_1071, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-4-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-6-7 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RSProcedureDispatcher-pool-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-6-4 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:62) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:883) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/089ee269-8889-d2f6-f8a2-4bb8eac365fc/cluster_a7d76481-495d-c9f6-361f-c7016573445a/data/data3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-6-6 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:62) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:883) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_OPEN_REGION-regionserver/1ab588ae992e:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_CLOSE_REGION-regionserver/1ab588ae992e:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=1050 (was 1024) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=217 (was 228), ProcessCount=11 (was 11), AvailableMemoryMB=2124 (was 2184) 2024-11-24T07:35:32,686 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1050 is superior to 1024 2024-11-24T07:35:32,697 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestAsyncWALReplayValueCompression#testReplayEditsAfterPartialFlush Thread=413, OpenFileDescriptor=1050, MaxFileDescriptor=1048576, SystemLoadAverage=217, ProcessCount=11, AvailableMemoryMB=2124 2024-11-24T07:35:32,697 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1050 is superior to 1024 2024-11-24T07:35:32,715 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-24T07:35:32,717 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-24T07:35:32,717 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-24T07:35:32,720 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-87630122, suffix=, logDir=hdfs://localhost:33505/hbase/WALs/hregion-87630122, archiveDir=hdfs://localhost:33505/hbase/oldWALs, maxLogs=32 2024-11-24T07:35:32,732 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-87630122/hregion-87630122.1732433732720, exclude list is [], retry=0 2024-11-24T07:35:32,735 DEBUG [AsyncFSWAL-20-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41161,DS-b0260eac-1380-4e84-bf22-a60109fdf5f6,DISK] 2024-11-24T07:35:32,735 DEBUG [AsyncFSWAL-20-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:42685,DS-467ee9a1-f3ca-4279-9b61-69a2a6a0308f,DISK] 2024-11-24T07:35:32,736 DEBUG [AsyncFSWAL-20-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41971,DS-a67a12dc-5f72-4d11-ac5a-3e225e33c62a,DISK] 2024-11-24T07:35:32,739 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-87630122/hregion-87630122.1732433732720 2024-11-24T07:35:32,739 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:40647:40647),(127.0.0.1/127.0.0.1:35537:35537),(127.0.0.1/127.0.0.1:37111:37111)] 2024-11-24T07:35:32,739 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => add2d09d5cb7429185a0d47a697a06c1, NAME => 'testReplayEditsWrittenViaHRegion,,1732433732716.add2d09d5cb7429185a0d47a697a06c1.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testReplayEditsWrittenViaHRegion', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:33505/hbase 2024-11-24T07:35:32,753 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41161 is added to blk_1073741898_1076 (size=67) 2024-11-24T07:35:32,753 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42685 is added to blk_1073741898_1076 (size=67) 2024-11-24T07:35:32,753 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41971 is added to blk_1073741898_1076 (size=67) 2024-11-24T07:35:32,754 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenViaHRegion,,1732433732716.add2d09d5cb7429185a0d47a697a06c1.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T07:35:32,756 INFO [StoreOpener-add2d09d5cb7429185a0d47a697a06c1-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region add2d09d5cb7429185a0d47a697a06c1 2024-11-24T07:35:32,757 INFO [StoreOpener-add2d09d5cb7429185a0d47a697a06c1-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region add2d09d5cb7429185a0d47a697a06c1 columnFamilyName a 2024-11-24T07:35:32,758 DEBUG [StoreOpener-add2d09d5cb7429185a0d47a697a06c1-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T07:35:32,758 INFO [StoreOpener-add2d09d5cb7429185a0d47a697a06c1-1 {}] regionserver.HStore(327): Store=add2d09d5cb7429185a0d47a697a06c1/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T07:35:32,758 INFO [StoreOpener-add2d09d5cb7429185a0d47a697a06c1-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region add2d09d5cb7429185a0d47a697a06c1 2024-11-24T07:35:32,760 INFO [StoreOpener-add2d09d5cb7429185a0d47a697a06c1-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region add2d09d5cb7429185a0d47a697a06c1 columnFamilyName b 2024-11-24T07:35:32,760 DEBUG [StoreOpener-add2d09d5cb7429185a0d47a697a06c1-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T07:35:32,761 INFO [StoreOpener-add2d09d5cb7429185a0d47a697a06c1-1 {}] regionserver.HStore(327): Store=add2d09d5cb7429185a0d47a697a06c1/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T07:35:32,761 INFO [StoreOpener-add2d09d5cb7429185a0d47a697a06c1-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region add2d09d5cb7429185a0d47a697a06c1 2024-11-24T07:35:32,763 INFO [StoreOpener-add2d09d5cb7429185a0d47a697a06c1-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region add2d09d5cb7429185a0d47a697a06c1 columnFamilyName c 2024-11-24T07:35:32,763 DEBUG [StoreOpener-add2d09d5cb7429185a0d47a697a06c1-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T07:35:32,763 INFO [StoreOpener-add2d09d5cb7429185a0d47a697a06c1-1 {}] regionserver.HStore(327): Store=add2d09d5cb7429185a0d47a697a06c1/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T07:35:32,763 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for add2d09d5cb7429185a0d47a697a06c1 2024-11-24T07:35:32,764 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33505/hbase/data/default/testReplayEditsWrittenViaHRegion/add2d09d5cb7429185a0d47a697a06c1 2024-11-24T07:35:32,764 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33505/hbase/data/default/testReplayEditsWrittenViaHRegion/add2d09d5cb7429185a0d47a697a06c1 2024-11-24T07:35:32,765 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for add2d09d5cb7429185a0d47a697a06c1 2024-11-24T07:35:32,765 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for add2d09d5cb7429185a0d47a697a06c1 2024-11-24T07:35:32,766 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsWrittenViaHRegion descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-24T07:35:32,767 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for add2d09d5cb7429185a0d47a697a06c1 2024-11-24T07:35:32,769 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:33505/hbase/data/default/testReplayEditsWrittenViaHRegion/add2d09d5cb7429185a0d47a697a06c1/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-24T07:35:32,770 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened add2d09d5cb7429185a0d47a697a06c1; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=66966360, jitterRate=-0.0021234750747680664}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-24T07:35:32,770 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for add2d09d5cb7429185a0d47a697a06c1: Writing region info on filesystem at 1732433732754Initializing all the Stores at 1732433732755 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732433732755Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732433732755Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732433732755Cleaning up temporary data from old regions at 1732433732765 (+10 ms)Region opened successfully at 1732433732770 (+5 ms) 2024-11-24T07:35:32,770 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing add2d09d5cb7429185a0d47a697a06c1, disabling compactions & flushes 2024-11-24T07:35:32,771 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsWrittenViaHRegion,,1732433732716.add2d09d5cb7429185a0d47a697a06c1. 2024-11-24T07:35:32,771 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsWrittenViaHRegion,,1732433732716.add2d09d5cb7429185a0d47a697a06c1. 2024-11-24T07:35:32,771 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsWrittenViaHRegion,,1732433732716.add2d09d5cb7429185a0d47a697a06c1. after waiting 0 ms 2024-11-24T07:35:32,771 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsWrittenViaHRegion,,1732433732716.add2d09d5cb7429185a0d47a697a06c1. 2024-11-24T07:35:32,771 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsWrittenViaHRegion,,1732433732716.add2d09d5cb7429185a0d47a697a06c1. 2024-11-24T07:35:32,771 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for add2d09d5cb7429185a0d47a697a06c1: Waiting for close lock at 1732433732770Disabling compacts and flushes for region at 1732433732770Disabling writes for close at 1732433732771 (+1 ms)Writing region close event to WAL at 1732433732771Closed at 1732433732771 2024-11-24T07:35:32,774 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41971 is added to blk_1073741897_1075 (size=95) 2024-11-24T07:35:32,774 WARN [Close-WAL-Writer-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(650): complete file /hbase/WALs/hregion-87630122/hregion-87630122.1732433732720 not finished, retry = 0 2024-11-24T07:35:32,774 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41161 is added to blk_1073741897_1075 (size=95) 2024-11-24T07:35:32,774 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42685 is added to blk_1073741897_1075 (size=95) 2024-11-24T07:35:32,879 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-11-24T07:35:32,879 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-87630122:(num 1732433732720) 2024-11-24T07:35:32,879 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-24T07:35:32,883 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:33505/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1732433732714, archiveDir=hdfs://localhost:33505/hbase/oldWALs, maxLogs=32 2024-11-24T07:35:32,898 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1732433732714/wal.1732433732884, exclude list is [], retry=0 2024-11-24T07:35:32,901 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41971,DS-a67a12dc-5f72-4d11-ac5a-3e225e33c62a,DISK] 2024-11-24T07:35:32,901 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41161,DS-b0260eac-1380-4e84-bf22-a60109fdf5f6,DISK] 2024-11-24T07:35:32,902 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:42685,DS-467ee9a1-f3ca-4279-9b61-69a2a6a0308f,DISK] 2024-11-24T07:35:32,903 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1732433732714/wal.1732433732884 2024-11-24T07:35:32,903 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:37111:37111),(127.0.0.1/127.0.0.1:40647:40647),(127.0.0.1/127.0.0.1:35537:35537)] 2024-11-24T07:35:32,903 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => add2d09d5cb7429185a0d47a697a06c1, NAME => 'testReplayEditsWrittenViaHRegion,,1732433732716.add2d09d5cb7429185a0d47a697a06c1.', STARTKEY => '', ENDKEY => ''} 2024-11-24T07:35:32,904 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenViaHRegion,,1732433732716.add2d09d5cb7429185a0d47a697a06c1.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T07:35:32,904 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for add2d09d5cb7429185a0d47a697a06c1 2024-11-24T07:35:32,904 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for add2d09d5cb7429185a0d47a697a06c1 2024-11-24T07:35:32,905 INFO [StoreOpener-add2d09d5cb7429185a0d47a697a06c1-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region add2d09d5cb7429185a0d47a697a06c1 2024-11-24T07:35:32,906 INFO [StoreOpener-add2d09d5cb7429185a0d47a697a06c1-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region add2d09d5cb7429185a0d47a697a06c1 columnFamilyName a 2024-11-24T07:35:32,906 DEBUG [StoreOpener-add2d09d5cb7429185a0d47a697a06c1-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T07:35:32,907 INFO [StoreOpener-add2d09d5cb7429185a0d47a697a06c1-1 {}] regionserver.HStore(327): Store=add2d09d5cb7429185a0d47a697a06c1/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T07:35:32,907 INFO [StoreOpener-add2d09d5cb7429185a0d47a697a06c1-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region add2d09d5cb7429185a0d47a697a06c1 2024-11-24T07:35:32,907 INFO [StoreOpener-add2d09d5cb7429185a0d47a697a06c1-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region add2d09d5cb7429185a0d47a697a06c1 columnFamilyName b 2024-11-24T07:35:32,908 DEBUG [StoreOpener-add2d09d5cb7429185a0d47a697a06c1-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T07:35:32,908 INFO [StoreOpener-add2d09d5cb7429185a0d47a697a06c1-1 {}] regionserver.HStore(327): Store=add2d09d5cb7429185a0d47a697a06c1/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T07:35:32,908 INFO [StoreOpener-add2d09d5cb7429185a0d47a697a06c1-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region add2d09d5cb7429185a0d47a697a06c1 2024-11-24T07:35:32,909 INFO [StoreOpener-add2d09d5cb7429185a0d47a697a06c1-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region add2d09d5cb7429185a0d47a697a06c1 columnFamilyName c 2024-11-24T07:35:32,909 DEBUG [StoreOpener-add2d09d5cb7429185a0d47a697a06c1-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T07:35:32,909 INFO [StoreOpener-add2d09d5cb7429185a0d47a697a06c1-1 {}] regionserver.HStore(327): Store=add2d09d5cb7429185a0d47a697a06c1/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T07:35:32,909 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for add2d09d5cb7429185a0d47a697a06c1 2024-11-24T07:35:32,910 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33505/hbase/data/default/testReplayEditsWrittenViaHRegion/add2d09d5cb7429185a0d47a697a06c1 2024-11-24T07:35:32,911 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33505/hbase/data/default/testReplayEditsWrittenViaHRegion/add2d09d5cb7429185a0d47a697a06c1 2024-11-24T07:35:32,912 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for add2d09d5cb7429185a0d47a697a06c1 2024-11-24T07:35:32,912 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for add2d09d5cb7429185a0d47a697a06c1 2024-11-24T07:35:32,913 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsWrittenViaHRegion descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-24T07:35:32,914 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for add2d09d5cb7429185a0d47a697a06c1 2024-11-24T07:35:32,915 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened add2d09d5cb7429185a0d47a697a06c1; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=71818946, jitterRate=0.07018569111824036}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-24T07:35:32,916 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for add2d09d5cb7429185a0d47a697a06c1: Writing region info on filesystem at 1732433732904Initializing all the Stores at 1732433732905 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732433732905Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732433732905Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732433732905Cleaning up temporary data from old regions at 1732433732912 (+7 ms)Region opened successfully at 1732433732916 (+4 ms) 2024-11-24T07:35:32,940 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing add2d09d5cb7429185a0d47a697a06c1 3/3 column families, dataSize=2.55 KB heapSize=5.44 KB 2024-11-24T07:35:32,955 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33505/hbase/data/default/testReplayEditsWrittenViaHRegion/add2d09d5cb7429185a0d47a697a06c1/.tmp/a/b833982d168d41a2a2fd5e461ae8e915 is 91, key is testReplayEditsWrittenViaHRegion/a:x0/1732433732916/Put/seqid=0 2024-11-24T07:35:32,965 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41161 is added to blk_1073741900_1078 (size=5958) 2024-11-24T07:35:32,965 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41971 is added to blk_1073741900_1078 (size=5958) 2024-11-24T07:35:32,965 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42685 is added to blk_1073741900_1078 (size=5958) 2024-11-24T07:35:32,966 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=33 (bloomFilter=true), to=hdfs://localhost:33505/hbase/data/default/testReplayEditsWrittenViaHRegion/add2d09d5cb7429185a0d47a697a06c1/.tmp/a/b833982d168d41a2a2fd5e461ae8e915 2024-11-24T07:35:32,984 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33505/hbase/data/default/testReplayEditsWrittenViaHRegion/add2d09d5cb7429185a0d47a697a06c1/.tmp/b/ca4a996523284e8eb2c8d3c316c3c91b is 91, key is testReplayEditsWrittenViaHRegion/b:x0/1732433732925/Put/seqid=0 2024-11-24T07:35:32,991 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41161 is added to blk_1073741901_1079 (size=5958) 2024-11-24T07:35:32,992 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41971 is added to blk_1073741901_1079 (size=5958) 2024-11-24T07:35:32,992 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42685 is added to blk_1073741901_1079 (size=5958) 2024-11-24T07:35:32,992 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=33 (bloomFilter=true), to=hdfs://localhost:33505/hbase/data/default/testReplayEditsWrittenViaHRegion/add2d09d5cb7429185a0d47a697a06c1/.tmp/b/ca4a996523284e8eb2c8d3c316c3c91b 2024-11-24T07:35:33,011 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33505/hbase/data/default/testReplayEditsWrittenViaHRegion/add2d09d5cb7429185a0d47a697a06c1/.tmp/c/55456f4ae50c4657a7332e39665434c7 is 91, key is testReplayEditsWrittenViaHRegion/c:x0/1732433732932/Put/seqid=0 2024-11-24T07:35:33,017 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42685 is added to blk_1073741902_1080 (size=5958) 2024-11-24T07:35:33,018 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41161 is added to blk_1073741902_1080 (size=5958) 2024-11-24T07:35:33,018 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41971 is added to blk_1073741902_1080 (size=5958) 2024-11-24T07:35:33,018 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=33 (bloomFilter=true), to=hdfs://localhost:33505/hbase/data/default/testReplayEditsWrittenViaHRegion/add2d09d5cb7429185a0d47a697a06c1/.tmp/c/55456f4ae50c4657a7332e39665434c7 2024-11-24T07:35:33,024 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33505/hbase/data/default/testReplayEditsWrittenViaHRegion/add2d09d5cb7429185a0d47a697a06c1/.tmp/a/b833982d168d41a2a2fd5e461ae8e915 as hdfs://localhost:33505/hbase/data/default/testReplayEditsWrittenViaHRegion/add2d09d5cb7429185a0d47a697a06c1/a/b833982d168d41a2a2fd5e461ae8e915 2024-11-24T07:35:33,030 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33505/hbase/data/default/testReplayEditsWrittenViaHRegion/add2d09d5cb7429185a0d47a697a06c1/a/b833982d168d41a2a2fd5e461ae8e915, entries=10, sequenceid=33, filesize=5.8 K 2024-11-24T07:35:33,031 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33505/hbase/data/default/testReplayEditsWrittenViaHRegion/add2d09d5cb7429185a0d47a697a06c1/.tmp/b/ca4a996523284e8eb2c8d3c316c3c91b as hdfs://localhost:33505/hbase/data/default/testReplayEditsWrittenViaHRegion/add2d09d5cb7429185a0d47a697a06c1/b/ca4a996523284e8eb2c8d3c316c3c91b 2024-11-24T07:35:33,036 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33505/hbase/data/default/testReplayEditsWrittenViaHRegion/add2d09d5cb7429185a0d47a697a06c1/b/ca4a996523284e8eb2c8d3c316c3c91b, entries=10, sequenceid=33, filesize=5.8 K 2024-11-24T07:35:33,038 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33505/hbase/data/default/testReplayEditsWrittenViaHRegion/add2d09d5cb7429185a0d47a697a06c1/.tmp/c/55456f4ae50c4657a7332e39665434c7 as hdfs://localhost:33505/hbase/data/default/testReplayEditsWrittenViaHRegion/add2d09d5cb7429185a0d47a697a06c1/c/55456f4ae50c4657a7332e39665434c7 2024-11-24T07:35:33,043 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33505/hbase/data/default/testReplayEditsWrittenViaHRegion/add2d09d5cb7429185a0d47a697a06c1/c/55456f4ae50c4657a7332e39665434c7, entries=10, sequenceid=33, filesize=5.8 K 2024-11-24T07:35:33,044 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~2.55 KB/2610, heapSize ~5.39 KB/5520, currentSize=0 B/0 for add2d09d5cb7429185a0d47a697a06c1 in 104ms, sequenceid=33, compaction requested=false 2024-11-24T07:35:33,044 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for add2d09d5cb7429185a0d47a697a06c1: 2024-11-24T07:35:33,044 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing add2d09d5cb7429185a0d47a697a06c1, disabling compactions & flushes 2024-11-24T07:35:33,045 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsWrittenViaHRegion,,1732433732716.add2d09d5cb7429185a0d47a697a06c1. 2024-11-24T07:35:33,045 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsWrittenViaHRegion,,1732433732716.add2d09d5cb7429185a0d47a697a06c1. 2024-11-24T07:35:33,045 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsWrittenViaHRegion,,1732433732716.add2d09d5cb7429185a0d47a697a06c1. after waiting 0 ms 2024-11-24T07:35:33,045 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsWrittenViaHRegion,,1732433732716.add2d09d5cb7429185a0d47a697a06c1. 2024-11-24T07:35:33,046 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsWrittenViaHRegion,,1732433732716.add2d09d5cb7429185a0d47a697a06c1. 2024-11-24T07:35:33,046 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for add2d09d5cb7429185a0d47a697a06c1: Waiting for close lock at 1732433733044Disabling compacts and flushes for region at 1732433733044Disabling writes for close at 1732433733045 (+1 ms)Writing region close event to WAL at 1732433733046 (+1 ms)Closed at 1732433733046 2024-11-24T07:35:33,049 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41161 is added to blk_1073741899_1077 (size=3381) 2024-11-24T07:35:33,049 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42685 is added to blk_1073741899_1077 (size=3381) 2024-11-24T07:35:33,049 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41971 is added to blk_1073741899_1077 (size=3381) 2024-11-24T07:35:33,051 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testSequentialEditLogSeqNum 2024-11-24T07:35:33,051 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testSequentialEditLogSeqNum Metrics about Tables on a single HBase RegionServer 2024-11-24T07:35:33,052 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testReplayEditsAfterRegionMovedWithMultiCF 2024-11-24T07:35:33,052 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testReplayEditsAfterRegionMovedWithMultiCF Metrics about Tables on a single HBase RegionServer 2024-11-24T07:35:33,053 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testRegionMadeOfBulkLoadedFilesOnly 2024-11-24T07:35:33,053 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testRegionMadeOfBulkLoadedFilesOnly Metrics about Tables on a single HBase RegionServer 2024-11-24T07:35:33,056 DEBUG [Time-limited test {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33505/hbase/data/default/testReplayEditsWrittenViaHRegion/add2d09d5cb7429185a0d47a697a06c1/b/ca4a996523284e8eb2c8d3c316c3c91b to hdfs://localhost:33505/hbase/archive/data/default/testReplayEditsWrittenViaHRegion/add2d09d5cb7429185a0d47a697a06c1/b/ca4a996523284e8eb2c8d3c316c3c91b 2024-11-24T07:35:33,079 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:33505/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1732433732714/wal.1732433732884, size=3.3 K (3381bytes) 2024-11-24T07:35:33,079 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:33505/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1732433732714/wal.1732433732884 2024-11-24T07:35:33,080 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:33505/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1732433732714/wal.1732433732884 after 1ms 2024-11-24T07:35:33,082 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:33505/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1732433732714/wal.1732433732884: isRecoveredEdits=false, hasTagCompression=true, hasValueCompression=true, valueCompressionType=GZ 2024-11-24T07:35:33,083 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:33505/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1732433732714/wal.1732433732884 took 3ms 2024-11-24T07:35:33,086 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:33505/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1732433732714/wal.1732433732884 so closing down 2024-11-24T07:35:33,086 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-24T07:35:33,086 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000003-wal.1732433732884.temp 2024-11-24T07:35:33,087 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:33505/hbase/data/default/testReplayEditsWrittenViaHRegion/add2d09d5cb7429185a0d47a697a06c1/recovered.edits/0000000000000000003-wal.1732433732884.temp 2024-11-24T07:35:33,088 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-11-24T07:35:33,098 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41971 is added to blk_1073741903_1081 (size=2944) 2024-11-24T07:35:33,098 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42685 is added to blk_1073741903_1081 (size=2944) 2024-11-24T07:35:33,098 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41161 is added to blk_1073741903_1081 (size=2944) 2024-11-24T07:35:33,100 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:33505/hbase/data/default/testReplayEditsWrittenViaHRegion/add2d09d5cb7429185a0d47a697a06c1/recovered.edits/0000000000000000003-wal.1732433732884.temp (wrote 30 edits, skipped 0 edits in 0 ms) 2024-11-24T07:35:33,101 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:33505/hbase/data/default/testReplayEditsWrittenViaHRegion/add2d09d5cb7429185a0d47a697a06c1/recovered.edits/0000000000000000003-wal.1732433732884.temp to hdfs://localhost:33505/hbase/data/default/testReplayEditsWrittenViaHRegion/add2d09d5cb7429185a0d47a697a06c1/recovered.edits/0000000000000000032 2024-11-24T07:35:33,102 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 32 edits across 1 Regions in 18 ms; skipped=2; WAL=hdfs://localhost:33505/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1732433732714/wal.1732433732884, size=3.3 K, length=3381, corrupted=false, cancelled=false 2024-11-24T07:35:33,102 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:33505/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1732433732714/wal.1732433732884, journal: Splitting hdfs://localhost:33505/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1732433732714/wal.1732433732884, size=3.3 K (3381bytes) at 1732433733079Finishing writing output for hdfs://localhost:33505/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1732433732714/wal.1732433732884 so closing down at 1732433733086 (+7 ms)Creating recovered edits writer path=hdfs://localhost:33505/hbase/data/default/testReplayEditsWrittenViaHRegion/add2d09d5cb7429185a0d47a697a06c1/recovered.edits/0000000000000000003-wal.1732433732884.temp at 1732433733087 (+1 ms)3 split writer threads finished at 1732433733088 (+1 ms)Closed recovered edits writer path=hdfs://localhost:33505/hbase/data/default/testReplayEditsWrittenViaHRegion/add2d09d5cb7429185a0d47a697a06c1/recovered.edits/0000000000000000003-wal.1732433732884.temp (wrote 30 edits, skipped 0 edits in 0 ms) at 1732433733100 (+12 ms)Rename recovered edits hdfs://localhost:33505/hbase/data/default/testReplayEditsWrittenViaHRegion/add2d09d5cb7429185a0d47a697a06c1/recovered.edits/0000000000000000003-wal.1732433732884.temp to hdfs://localhost:33505/hbase/data/default/testReplayEditsWrittenViaHRegion/add2d09d5cb7429185a0d47a697a06c1/recovered.edits/0000000000000000032 at 1732433733101 (+1 ms)Processed 32 edits across 1 Regions in 18 ms; skipped=2; WAL=hdfs://localhost:33505/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1732433732714/wal.1732433732884, size=3.3 K, length=3381, corrupted=false, cancelled=false at 1732433733102 (+1 ms) 2024-11-24T07:35:33,104 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:33505/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1732433732714/wal.1732433732884 to hdfs://localhost:33505/hbase/oldWALs/wal.1732433732884 2024-11-24T07:35:33,105 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:33505/hbase/data/default/testReplayEditsWrittenViaHRegion/add2d09d5cb7429185a0d47a697a06c1/recovered.edits/0000000000000000032 2024-11-24T07:35:33,105 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-24T07:35:33,107 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:33505/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1732433732714, archiveDir=hdfs://localhost:33505/hbase/oldWALs, maxLogs=32 2024-11-24T07:35:33,127 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1732433732714/wal.1732433733108, exclude list is [], retry=0 2024-11-24T07:35:33,131 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:42685,DS-467ee9a1-f3ca-4279-9b61-69a2a6a0308f,DISK] 2024-11-24T07:35:33,131 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41161,DS-b0260eac-1380-4e84-bf22-a60109fdf5f6,DISK] 2024-11-24T07:35:33,132 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41971,DS-a67a12dc-5f72-4d11-ac5a-3e225e33c62a,DISK] 2024-11-24T07:35:33,134 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1732433732714/wal.1732433733108 2024-11-24T07:35:33,135 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:35537:35537),(127.0.0.1/127.0.0.1:40647:40647),(127.0.0.1/127.0.0.1:37111:37111)] 2024-11-24T07:35:33,135 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => add2d09d5cb7429185a0d47a697a06c1, NAME => 'testReplayEditsWrittenViaHRegion,,1732433732716.add2d09d5cb7429185a0d47a697a06c1.', STARTKEY => '', ENDKEY => ''} 2024-11-24T07:35:33,135 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenViaHRegion,,1732433732716.add2d09d5cb7429185a0d47a697a06c1.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T07:35:33,135 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for add2d09d5cb7429185a0d47a697a06c1 2024-11-24T07:35:33,135 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for add2d09d5cb7429185a0d47a697a06c1 2024-11-24T07:35:33,137 INFO [StoreOpener-add2d09d5cb7429185a0d47a697a06c1-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region add2d09d5cb7429185a0d47a697a06c1 2024-11-24T07:35:33,138 INFO [StoreOpener-add2d09d5cb7429185a0d47a697a06c1-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region add2d09d5cb7429185a0d47a697a06c1 columnFamilyName a 2024-11-24T07:35:33,138 DEBUG [StoreOpener-add2d09d5cb7429185a0d47a697a06c1-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T07:35:33,145 DEBUG [StoreOpener-add2d09d5cb7429185a0d47a697a06c1-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:33505/hbase/data/default/testReplayEditsWrittenViaHRegion/add2d09d5cb7429185a0d47a697a06c1/a/b833982d168d41a2a2fd5e461ae8e915 2024-11-24T07:35:33,145 INFO [StoreOpener-add2d09d5cb7429185a0d47a697a06c1-1 {}] regionserver.HStore(327): Store=add2d09d5cb7429185a0d47a697a06c1/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T07:35:33,146 INFO [StoreOpener-add2d09d5cb7429185a0d47a697a06c1-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region add2d09d5cb7429185a0d47a697a06c1 2024-11-24T07:35:33,148 INFO [StoreOpener-add2d09d5cb7429185a0d47a697a06c1-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region add2d09d5cb7429185a0d47a697a06c1 columnFamilyName b 2024-11-24T07:35:33,148 DEBUG [StoreOpener-add2d09d5cb7429185a0d47a697a06c1-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T07:35:33,149 INFO [StoreOpener-add2d09d5cb7429185a0d47a697a06c1-1 {}] regionserver.HStore(327): Store=add2d09d5cb7429185a0d47a697a06c1/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T07:35:33,149 INFO [StoreOpener-add2d09d5cb7429185a0d47a697a06c1-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region add2d09d5cb7429185a0d47a697a06c1 2024-11-24T07:35:33,150 INFO [StoreOpener-add2d09d5cb7429185a0d47a697a06c1-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region add2d09d5cb7429185a0d47a697a06c1 columnFamilyName c 2024-11-24T07:35:33,150 DEBUG [StoreOpener-add2d09d5cb7429185a0d47a697a06c1-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T07:35:33,159 DEBUG [StoreOpener-add2d09d5cb7429185a0d47a697a06c1-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:33505/hbase/data/default/testReplayEditsWrittenViaHRegion/add2d09d5cb7429185a0d47a697a06c1/c/55456f4ae50c4657a7332e39665434c7 2024-11-24T07:35:33,159 INFO [StoreOpener-add2d09d5cb7429185a0d47a697a06c1-1 {}] regionserver.HStore(327): Store=add2d09d5cb7429185a0d47a697a06c1/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T07:35:33,159 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for add2d09d5cb7429185a0d47a697a06c1 2024-11-24T07:35:33,160 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33505/hbase/data/default/testReplayEditsWrittenViaHRegion/add2d09d5cb7429185a0d47a697a06c1 2024-11-24T07:35:33,162 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:33505/hbase/data/default/testReplayEditsWrittenViaHRegion/add2d09d5cb7429185a0d47a697a06c1 2024-11-24T07:35:33,162 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:33505/hbase/data/default/testReplayEditsWrittenViaHRegion/add2d09d5cb7429185a0d47a697a06c1/recovered.edits/0000000000000000032 2024-11-24T07:35:33,165 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:33505/hbase/data/default/testReplayEditsWrittenViaHRegion/add2d09d5cb7429185a0d47a697a06c1/recovered.edits/0000000000000000032: isRecoveredEdits=true, hasTagCompression=true, hasValueCompression=true, valueCompressionType=GZ 2024-11-24T07:35:33,167 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 10, skipped 20, firstSequenceIdInLog=3, maxSequenceIdInLog=32, path=hdfs://localhost:33505/hbase/data/default/testReplayEditsWrittenViaHRegion/add2d09d5cb7429185a0d47a697a06c1/recovered.edits/0000000000000000032 2024-11-24T07:35:33,167 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing add2d09d5cb7429185a0d47a697a06c1 3/3 column families, dataSize=870 B heapSize=2.31 KB 2024-11-24T07:35:33,188 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33505/hbase/data/default/testReplayEditsWrittenViaHRegion/add2d09d5cb7429185a0d47a697a06c1/.tmp/b/f981623cd0c2462582eeb5f91cc45b3a is 91, key is testReplayEditsWrittenViaHRegion/b:x0/1732433732925/Put/seqid=0 2024-11-24T07:35:33,195 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42685 is added to blk_1073741905_1083 (size=5958) 2024-11-24T07:35:33,195 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41971 is added to blk_1073741905_1083 (size=5958) 2024-11-24T07:35:33,195 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41161 is added to blk_1073741905_1083 (size=5958) 2024-11-24T07:35:33,196 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=32 (bloomFilter=true), to=hdfs://localhost:33505/hbase/data/default/testReplayEditsWrittenViaHRegion/add2d09d5cb7429185a0d47a697a06c1/.tmp/b/f981623cd0c2462582eeb5f91cc45b3a 2024-11-24T07:35:33,203 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33505/hbase/data/default/testReplayEditsWrittenViaHRegion/add2d09d5cb7429185a0d47a697a06c1/.tmp/b/f981623cd0c2462582eeb5f91cc45b3a as hdfs://localhost:33505/hbase/data/default/testReplayEditsWrittenViaHRegion/add2d09d5cb7429185a0d47a697a06c1/b/f981623cd0c2462582eeb5f91cc45b3a 2024-11-24T07:35:33,209 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33505/hbase/data/default/testReplayEditsWrittenViaHRegion/add2d09d5cb7429185a0d47a697a06c1/b/f981623cd0c2462582eeb5f91cc45b3a, entries=10, sequenceid=32, filesize=5.8 K 2024-11-24T07:35:33,209 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~870 B/870, heapSize ~1.80 KB/1840, currentSize=0 B/0 for add2d09d5cb7429185a0d47a697a06c1 in 42ms, sequenceid=32, compaction requested=false; wal=null 2024-11-24T07:35:33,210 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:33505/hbase/data/default/testReplayEditsWrittenViaHRegion/add2d09d5cb7429185a0d47a697a06c1/recovered.edits/0000000000000000032 2024-11-24T07:35:33,211 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for add2d09d5cb7429185a0d47a697a06c1 2024-11-24T07:35:33,211 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for add2d09d5cb7429185a0d47a697a06c1 2024-11-24T07:35:33,212 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsWrittenViaHRegion descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-24T07:35:33,214 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for add2d09d5cb7429185a0d47a697a06c1 2024-11-24T07:35:33,216 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:33505/hbase/data/default/testReplayEditsWrittenViaHRegion/add2d09d5cb7429185a0d47a697a06c1/recovered.edits/33.seqid, newMaxSeqId=33, maxSeqId=1 2024-11-24T07:35:33,217 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened add2d09d5cb7429185a0d47a697a06c1; next sequenceid=34; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=72922265, jitterRate=0.08662642538547516}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-24T07:35:33,218 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for add2d09d5cb7429185a0d47a697a06c1: Writing region info on filesystem at 1732433733135Initializing all the Stores at 1732433733136 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732433733137 (+1 ms)Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732433733137Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732433733137Obtaining lock to block concurrent updates at 1732433733167 (+30 ms)Preparing flush snapshotting stores in add2d09d5cb7429185a0d47a697a06c1 at 1732433733167Finished memstore snapshotting testReplayEditsWrittenViaHRegion,,1732433732716.add2d09d5cb7429185a0d47a697a06c1., syncing WAL and waiting on mvcc, flushsize=dataSize=870, getHeapSize=2320, getOffHeapSize=0, getCellsCount=10 at 1732433733167Flushing stores of testReplayEditsWrittenViaHRegion,,1732433732716.add2d09d5cb7429185a0d47a697a06c1. at 1732433733167Flushing add2d09d5cb7429185a0d47a697a06c1/b: creating writer at 1732433733167Flushing add2d09d5cb7429185a0d47a697a06c1/b: appending metadata at 1732433733187 (+20 ms)Flushing add2d09d5cb7429185a0d47a697a06c1/b: closing flushed file at 1732433733187Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@1a308bea: reopening flushed file at 1732433733202 (+15 ms)Finished flush of dataSize ~870 B/870, heapSize ~1.80 KB/1840, currentSize=0 B/0 for add2d09d5cb7429185a0d47a697a06c1 in 42ms, sequenceid=32, compaction requested=false; wal=null at 1732433733209 (+7 ms)Cleaning up temporary data from old regions at 1732433733211 (+2 ms)Region opened successfully at 1732433733217 (+6 ms) 2024-11-24T07:35:33,243 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestAsyncWALReplayValueCompression#testReplayEditsAfterPartialFlush Thread=422 (was 413) Potentially hanging thread: AsyncFSWAL-20-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-166876350_22 at /127.0.0.1:52922 [Waiting for operation #45] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-166876350_22 at /127.0.0.1:38060 [Receiving block BP-1337235608-172.17.0.3-1732433708251:blk_1073741904_1082] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-166876350_22 at /127.0.0.1:42132 [Receiving block BP-1337235608-172.17.0.3-1732433708251:blk_1073741904_1082] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-166876350_22 at /127.0.0.1:51590 [Waiting for operation #28] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-1337235608-172.17.0.3-1732433708251:blk_1073741904_1082, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-1337235608-172.17.0.3-1732433708251:blk_1073741904_1082, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-20-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-166876350_22 at /127.0.0.1:60130 [Waiting for operation #50] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-1337235608-172.17.0.3-1732433708251:blk_1073741904_1082, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-20-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-166876350_22 at /127.0.0.1:52148 [Receiving block BP-1337235608-172.17.0.3-1732433708251:blk_1073741904_1082] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=1126 (was 1050) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=217 (was 217), ProcessCount=11 (was 11), AvailableMemoryMB=2093 (was 2124) 2024-11-24T07:35:33,243 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1126 is superior to 1024 2024-11-24T07:35:33,257 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestAsyncWALReplayValueCompression#testReplayEditsAfterAbortingFlush Thread=422, OpenFileDescriptor=1126, MaxFileDescriptor=1048576, SystemLoadAverage=217, ProcessCount=11, AvailableMemoryMB=2091 2024-11-24T07:35:33,257 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1126 is superior to 1024 2024-11-24T07:35:33,272 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-24T07:35:33,275 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-24T07:35:33,276 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-24T07:35:33,279 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-35423104, suffix=, logDir=hdfs://localhost:33505/hbase/WALs/hregion-35423104, archiveDir=hdfs://localhost:33505/hbase/oldWALs, maxLogs=32 2024-11-24T07:35:33,292 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-35423104/hregion-35423104.1732433733279, exclude list is [], retry=0 2024-11-24T07:35:33,295 DEBUG [AsyncFSWAL-22-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41161,DS-b0260eac-1380-4e84-bf22-a60109fdf5f6,DISK] 2024-11-24T07:35:33,295 DEBUG [AsyncFSWAL-22-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:42685,DS-467ee9a1-f3ca-4279-9b61-69a2a6a0308f,DISK] 2024-11-24T07:35:33,296 DEBUG [AsyncFSWAL-22-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41971,DS-a67a12dc-5f72-4d11-ac5a-3e225e33c62a,DISK] 2024-11-24T07:35:33,298 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-35423104/hregion-35423104.1732433733279 2024-11-24T07:35:33,299 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:40647:40647),(127.0.0.1/127.0.0.1:35537:35537),(127.0.0.1/127.0.0.1:37111:37111)] 2024-11-24T07:35:33,299 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => 06230e08714a24222d993f2f016169a6, NAME => 'testReplayEditsAfterAbortingFlush,,1732433733273.06230e08714a24222d993f2f016169a6.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testReplayEditsAfterAbortingFlush', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:33505/hbase 2024-11-24T07:35:33,309 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42685 is added to blk_1073741907_1085 (size=68) 2024-11-24T07:35:33,310 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41971 is added to blk_1073741907_1085 (size=68) 2024-11-24T07:35:33,310 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41161 is added to blk_1073741907_1085 (size=68) 2024-11-24T07:35:33,335 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta' 2024-11-24T07:35:33,711 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsAfterAbortingFlush,,1732433733273.06230e08714a24222d993f2f016169a6.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T07:35:33,714 INFO [StoreOpener-06230e08714a24222d993f2f016169a6-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 06230e08714a24222d993f2f016169a6 2024-11-24T07:35:33,717 INFO [StoreOpener-06230e08714a24222d993f2f016169a6-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 06230e08714a24222d993f2f016169a6 columnFamilyName a 2024-11-24T07:35:33,717 DEBUG [StoreOpener-06230e08714a24222d993f2f016169a6-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T07:35:33,718 INFO [StoreOpener-06230e08714a24222d993f2f016169a6-1 {}] regionserver.HStore(327): Store=06230e08714a24222d993f2f016169a6/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T07:35:33,718 INFO [StoreOpener-06230e08714a24222d993f2f016169a6-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 06230e08714a24222d993f2f016169a6 2024-11-24T07:35:33,720 INFO [StoreOpener-06230e08714a24222d993f2f016169a6-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 06230e08714a24222d993f2f016169a6 columnFamilyName b 2024-11-24T07:35:33,721 DEBUG [StoreOpener-06230e08714a24222d993f2f016169a6-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T07:35:33,721 INFO [StoreOpener-06230e08714a24222d993f2f016169a6-1 {}] regionserver.HStore(327): Store=06230e08714a24222d993f2f016169a6/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T07:35:33,722 INFO [StoreOpener-06230e08714a24222d993f2f016169a6-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 06230e08714a24222d993f2f016169a6 2024-11-24T07:35:33,723 INFO [StoreOpener-06230e08714a24222d993f2f016169a6-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 06230e08714a24222d993f2f016169a6 columnFamilyName c 2024-11-24T07:35:33,724 DEBUG [StoreOpener-06230e08714a24222d993f2f016169a6-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T07:35:33,724 INFO [StoreOpener-06230e08714a24222d993f2f016169a6-1 {}] regionserver.HStore(327): Store=06230e08714a24222d993f2f016169a6/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T07:35:33,724 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 06230e08714a24222d993f2f016169a6 2024-11-24T07:35:33,726 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33505/hbase/data/default/testReplayEditsAfterAbortingFlush/06230e08714a24222d993f2f016169a6 2024-11-24T07:35:33,726 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33505/hbase/data/default/testReplayEditsAfterAbortingFlush/06230e08714a24222d993f2f016169a6 2024-11-24T07:35:33,728 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 06230e08714a24222d993f2f016169a6 2024-11-24T07:35:33,728 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 06230e08714a24222d993f2f016169a6 2024-11-24T07:35:33,728 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsAfterAbortingFlush descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-24T07:35:33,730 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 06230e08714a24222d993f2f016169a6 2024-11-24T07:35:33,733 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:33505/hbase/data/default/testReplayEditsAfterAbortingFlush/06230e08714a24222d993f2f016169a6/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-24T07:35:33,733 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 06230e08714a24222d993f2f016169a6; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=62198533, jitterRate=-0.07316963374614716}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-24T07:35:33,734 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 06230e08714a24222d993f2f016169a6: Writing region info on filesystem at 1732433733711Initializing all the Stores at 1732433733713 (+2 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732433733713Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732433733713Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732433733714 (+1 ms)Cleaning up temporary data from old regions at 1732433733728 (+14 ms)Region opened successfully at 1732433733734 (+6 ms) 2024-11-24T07:35:33,734 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 06230e08714a24222d993f2f016169a6, disabling compactions & flushes 2024-11-24T07:35:33,734 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsAfterAbortingFlush,,1732433733273.06230e08714a24222d993f2f016169a6. 2024-11-24T07:35:33,734 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsAfterAbortingFlush,,1732433733273.06230e08714a24222d993f2f016169a6. 2024-11-24T07:35:33,734 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsAfterAbortingFlush,,1732433733273.06230e08714a24222d993f2f016169a6. after waiting 0 ms 2024-11-24T07:35:33,734 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsAfterAbortingFlush,,1732433733273.06230e08714a24222d993f2f016169a6. 2024-11-24T07:35:33,734 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsAfterAbortingFlush,,1732433733273.06230e08714a24222d993f2f016169a6. 2024-11-24T07:35:33,734 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 06230e08714a24222d993f2f016169a6: Waiting for close lock at 1732433733734Disabling compacts and flushes for region at 1732433733734Disabling writes for close at 1732433733734Writing region close event to WAL at 1732433733734Closed at 1732433733734 2024-11-24T07:35:33,736 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41161 is added to blk_1073741906_1084 (size=95) 2024-11-24T07:35:33,737 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41971 is added to blk_1073741906_1084 (size=95) 2024-11-24T07:35:33,737 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42685 is added to blk_1073741906_1084 (size=95) 2024-11-24T07:35:33,738 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-11-24T07:35:33,738 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-35423104:(num 1732433733279) 2024-11-24T07:35:33,739 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-24T07:35:33,740 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:33505/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1732433733272, archiveDir=hdfs://localhost:33505/hbase/oldWALs, maxLogs=32 2024-11-24T07:35:33,754 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1732433733272/wal.1732433733741, exclude list is [], retry=0 2024-11-24T07:35:33,757 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:42685,DS-467ee9a1-f3ca-4279-9b61-69a2a6a0308f,DISK] 2024-11-24T07:35:33,757 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41161,DS-b0260eac-1380-4e84-bf22-a60109fdf5f6,DISK] 2024-11-24T07:35:33,757 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41971,DS-a67a12dc-5f72-4d11-ac5a-3e225e33c62a,DISK] 2024-11-24T07:35:33,759 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1732433733272/wal.1732433733741 2024-11-24T07:35:33,759 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:35537:35537),(127.0.0.1/127.0.0.1:40647:40647),(127.0.0.1/127.0.0.1:37111:37111)] 2024-11-24T07:35:33,832 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 06230e08714a24222d993f2f016169a6, NAME => 'testReplayEditsAfterAbortingFlush,,1732433733273.06230e08714a24222d993f2f016169a6.', STARTKEY => '', ENDKEY => ''} 2024-11-24T07:35:33,834 DEBUG [Time-limited test {}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testReplayEditsAfterAbortingFlush 06230e08714a24222d993f2f016169a6 2024-11-24T07:35:33,834 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsAfterAbortingFlush,,1732433733273.06230e08714a24222d993f2f016169a6.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T07:35:33,835 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 06230e08714a24222d993f2f016169a6 2024-11-24T07:35:33,835 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 06230e08714a24222d993f2f016169a6 2024-11-24T07:35:33,836 INFO [StoreOpener-06230e08714a24222d993f2f016169a6-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 06230e08714a24222d993f2f016169a6 2024-11-24T07:35:33,837 INFO [StoreOpener-06230e08714a24222d993f2f016169a6-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 06230e08714a24222d993f2f016169a6 columnFamilyName a 2024-11-24T07:35:33,837 DEBUG [StoreOpener-06230e08714a24222d993f2f016169a6-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T07:35:33,838 INFO [StoreOpener-06230e08714a24222d993f2f016169a6-1 {}] regionserver.HStore(327): Store=06230e08714a24222d993f2f016169a6/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T07:35:33,838 INFO [StoreOpener-06230e08714a24222d993f2f016169a6-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 06230e08714a24222d993f2f016169a6 2024-11-24T07:35:33,839 INFO [StoreOpener-06230e08714a24222d993f2f016169a6-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 06230e08714a24222d993f2f016169a6 columnFamilyName b 2024-11-24T07:35:33,839 DEBUG [StoreOpener-06230e08714a24222d993f2f016169a6-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T07:35:33,839 INFO [StoreOpener-06230e08714a24222d993f2f016169a6-1 {}] regionserver.HStore(327): Store=06230e08714a24222d993f2f016169a6/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T07:35:33,840 INFO [StoreOpener-06230e08714a24222d993f2f016169a6-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 06230e08714a24222d993f2f016169a6 2024-11-24T07:35:33,840 INFO [StoreOpener-06230e08714a24222d993f2f016169a6-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 06230e08714a24222d993f2f016169a6 columnFamilyName c 2024-11-24T07:35:33,841 DEBUG [StoreOpener-06230e08714a24222d993f2f016169a6-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T07:35:33,841 INFO [StoreOpener-06230e08714a24222d993f2f016169a6-1 {}] regionserver.HStore(327): Store=06230e08714a24222d993f2f016169a6/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T07:35:33,841 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 06230e08714a24222d993f2f016169a6 2024-11-24T07:35:33,842 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33505/hbase/data/default/testReplayEditsAfterAbortingFlush/06230e08714a24222d993f2f016169a6 2024-11-24T07:35:33,844 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33505/hbase/data/default/testReplayEditsAfterAbortingFlush/06230e08714a24222d993f2f016169a6 2024-11-24T07:35:33,845 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 06230e08714a24222d993f2f016169a6 2024-11-24T07:35:33,845 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 06230e08714a24222d993f2f016169a6 2024-11-24T07:35:33,846 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsAfterAbortingFlush descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-24T07:35:33,847 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 06230e08714a24222d993f2f016169a6 2024-11-24T07:35:33,848 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 06230e08714a24222d993f2f016169a6; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=71975669, jitterRate=0.07252104580402374}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-24T07:35:33,848 DEBUG [Time-limited test {}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 06230e08714a24222d993f2f016169a6 2024-11-24T07:35:33,849 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 06230e08714a24222d993f2f016169a6: Running coprocessor pre-open hook at 1732433733835Writing region info on filesystem at 1732433733835Initializing all the Stores at 1732433733836 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732433733836Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732433733836Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732433733836Cleaning up temporary data from old regions at 1732433733845 (+9 ms)Running coprocessor post-open hooks at 1732433733848 (+3 ms)Region opened successfully at 1732433733849 (+1 ms) 2024-11-24T07:35:33,861 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 06230e08714a24222d993f2f016169a6 3/3 column families, dataSize=590 B heapSize=2.08 KB 2024-11-24T07:35:33,863 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for 06230e08714a24222d993f2f016169a6/a, retrying num=0 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T07:35:33,996 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-24T07:35:34,864 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for 06230e08714a24222d993f2f016169a6/a, retrying num=1 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T07:35:35,865 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for 06230e08714a24222d993f2f016169a6/a, retrying num=2 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T07:35:36,866 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for 06230e08714a24222d993f2f016169a6/a, retrying num=3 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T07:35:37,868 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for 06230e08714a24222d993f2f016169a6/a, retrying num=4 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T07:35:38,870 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for 06230e08714a24222d993f2f016169a6/a, retrying num=5 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T07:35:39,341 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-24T07:35:39,871 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for 06230e08714a24222d993f2f016169a6/a, retrying num=6 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T07:35:40,873 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for 06230e08714a24222d993f2f016169a6/a, retrying num=7 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T07:35:41,873 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-24T07:35:41,874 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for 06230e08714a24222d993f2f016169a6/a, retrying num=8 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T07:35:42,875 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for 06230e08714a24222d993f2f016169a6/a, retrying num=9 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T07:35:42,877 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for 06230e08714a24222d993f2f016169a6: 2024-11-24T07:35:42,877 INFO [Time-limited test {}] wal.AbstractTestWALReplay(671): Expected simulated exception when flushing region, region: testReplayEditsAfterAbortingFlush,,1732433733273.06230e08714a24222d993f2f016169a6. 2024-11-24T07:35:42,895 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for 06230e08714a24222d993f2f016169a6: 2024-11-24T07:35:42,895 INFO [Time-limited test {}] wal.AbstractTestWALReplay(691): Expected exception when flushing region because server is stopped,Aborting flush because server is aborted... 2024-11-24T07:35:42,895 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 06230e08714a24222d993f2f016169a6, disabling compactions & flushes 2024-11-24T07:35:42,895 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsAfterAbortingFlush,,1732433733273.06230e08714a24222d993f2f016169a6. 2024-11-24T07:35:42,895 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsAfterAbortingFlush,,1732433733273.06230e08714a24222d993f2f016169a6. 2024-11-24T07:35:42,895 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsAfterAbortingFlush,,1732433733273.06230e08714a24222d993f2f016169a6. after waiting 0 ms 2024-11-24T07:35:42,895 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsAfterAbortingFlush,,1732433733273.06230e08714a24222d993f2f016169a6. 2024-11-24T07:35:42,896 ERROR [Time-limited test {}] regionserver.HRegion(1960): Memstore data size is 1190 in region testReplayEditsAfterAbortingFlush,,1732433733273.06230e08714a24222d993f2f016169a6. 2024-11-24T07:35:42,896 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsAfterAbortingFlush,,1732433733273.06230e08714a24222d993f2f016169a6. 2024-11-24T07:35:42,896 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 06230e08714a24222d993f2f016169a6: Waiting for close lock at 1732433742895Running coprocessor pre-close hooks at 1732433742895Disabling compacts and flushes for region at 1732433742895Disabling writes for close at 1732433742895Writing region close event to WAL at 1732433742896 (+1 ms)Running coprocessor post-close hooks at 1732433742896Closed at 1732433742896 2024-11-24T07:35:42,898 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41161 is added to blk_1073741908_1086 (size=2691) 2024-11-24T07:35:42,898 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41971 is added to blk_1073741908_1086 (size=2691) 2024-11-24T07:35:42,898 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42685 is added to blk_1073741908_1086 (size=2691) 2024-11-24T07:35:42,915 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:33505/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1732433733272/wal.1732433733741, size=2.6 K (2691bytes) 2024-11-24T07:35:42,915 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:33505/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1732433733272/wal.1732433733741 2024-11-24T07:35:42,916 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:33505/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1732433733272/wal.1732433733741 after 1ms 2024-11-24T07:35:42,918 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:33505/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1732433733272/wal.1732433733741: isRecoveredEdits=false, hasTagCompression=true, hasValueCompression=true, valueCompressionType=GZ 2024-11-24T07:35:42,919 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:33505/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1732433733272/wal.1732433733741 took 4ms 2024-11-24T07:35:42,921 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:33505/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1732433733272/wal.1732433733741 so closing down 2024-11-24T07:35:42,921 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-24T07:35:42,922 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000004-wal.1732433733741.temp 2024-11-24T07:35:42,923 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:33505/hbase/data/default/testReplayEditsAfterAbortingFlush/06230e08714a24222d993f2f016169a6/recovered.edits/0000000000000000004-wal.1732433733741.temp 2024-11-24T07:35:42,924 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-11-24T07:35:42,930 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41161 is added to blk_1073741909_1087 (size=2094) 2024-11-24T07:35:42,930 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41971 is added to blk_1073741909_1087 (size=2094) 2024-11-24T07:35:42,930 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42685 is added to blk_1073741909_1087 (size=2094) 2024-11-24T07:35:42,931 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:33505/hbase/data/default/testReplayEditsAfterAbortingFlush/06230e08714a24222d993f2f016169a6/recovered.edits/0000000000000000004-wal.1732433733741.temp (wrote 20 edits, skipped 0 edits in 0 ms) 2024-11-24T07:35:42,932 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:33505/hbase/data/default/testReplayEditsAfterAbortingFlush/06230e08714a24222d993f2f016169a6/recovered.edits/0000000000000000004-wal.1732433733741.temp to hdfs://localhost:33505/hbase/data/default/testReplayEditsAfterAbortingFlush/06230e08714a24222d993f2f016169a6/recovered.edits/0000000000000000026 2024-11-24T07:35:42,932 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 23 edits across 1 Regions in 13 ms; skipped=3; WAL=hdfs://localhost:33505/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1732433733272/wal.1732433733741, size=2.6 K, length=2691, corrupted=false, cancelled=false 2024-11-24T07:35:42,932 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:33505/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1732433733272/wal.1732433733741, journal: Splitting hdfs://localhost:33505/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1732433733272/wal.1732433733741, size=2.6 K (2691bytes) at 1732433742915Finishing writing output for hdfs://localhost:33505/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1732433733272/wal.1732433733741 so closing down at 1732433742921 (+6 ms)Creating recovered edits writer path=hdfs://localhost:33505/hbase/data/default/testReplayEditsAfterAbortingFlush/06230e08714a24222d993f2f016169a6/recovered.edits/0000000000000000004-wal.1732433733741.temp at 1732433742923 (+2 ms)3 split writer threads finished at 1732433742924 (+1 ms)Closed recovered edits writer path=hdfs://localhost:33505/hbase/data/default/testReplayEditsAfterAbortingFlush/06230e08714a24222d993f2f016169a6/recovered.edits/0000000000000000004-wal.1732433733741.temp (wrote 20 edits, skipped 0 edits in 0 ms) at 1732433742931 (+7 ms)Rename recovered edits hdfs://localhost:33505/hbase/data/default/testReplayEditsAfterAbortingFlush/06230e08714a24222d993f2f016169a6/recovered.edits/0000000000000000004-wal.1732433733741.temp to hdfs://localhost:33505/hbase/data/default/testReplayEditsAfterAbortingFlush/06230e08714a24222d993f2f016169a6/recovered.edits/0000000000000000026 at 1732433742932 (+1 ms)Processed 23 edits across 1 Regions in 13 ms; skipped=3; WAL=hdfs://localhost:33505/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1732433733272/wal.1732433733741, size=2.6 K, length=2691, corrupted=false, cancelled=false at 1732433742932 2024-11-24T07:35:42,934 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:33505/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1732433733272/wal.1732433733741 to hdfs://localhost:33505/hbase/oldWALs/wal.1732433733741 2024-11-24T07:35:42,935 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:33505/hbase/data/default/testReplayEditsAfterAbortingFlush/06230e08714a24222d993f2f016169a6/recovered.edits/0000000000000000026 2024-11-24T07:35:42,935 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-24T07:35:42,936 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:33505/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1732433733272, archiveDir=hdfs://localhost:33505/hbase/oldWALs, maxLogs=32 2024-11-24T07:35:42,949 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1732433733272/wal.1732433742937, exclude list is [], retry=0 2024-11-24T07:35:42,952 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41971,DS-a67a12dc-5f72-4d11-ac5a-3e225e33c62a,DISK] 2024-11-24T07:35:42,952 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:42685,DS-467ee9a1-f3ca-4279-9b61-69a2a6a0308f,DISK] 2024-11-24T07:35:42,952 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41161,DS-b0260eac-1380-4e84-bf22-a60109fdf5f6,DISK] 2024-11-24T07:35:42,954 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1732433733272/wal.1732433742937 2024-11-24T07:35:42,954 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:37111:37111),(127.0.0.1/127.0.0.1:35537:35537),(127.0.0.1/127.0.0.1:40647:40647)] 2024-11-24T07:35:42,955 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 06230e08714a24222d993f2f016169a6, NAME => 'testReplayEditsAfterAbortingFlush,,1732433733273.06230e08714a24222d993f2f016169a6.', STARTKEY => '', ENDKEY => ''} 2024-11-24T07:35:42,955 DEBUG [Time-limited test {}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testReplayEditsAfterAbortingFlush 06230e08714a24222d993f2f016169a6 2024-11-24T07:35:42,955 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsAfterAbortingFlush,,1732433733273.06230e08714a24222d993f2f016169a6.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T07:35:42,955 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 06230e08714a24222d993f2f016169a6 2024-11-24T07:35:42,955 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 06230e08714a24222d993f2f016169a6 2024-11-24T07:35:42,957 INFO [StoreOpener-06230e08714a24222d993f2f016169a6-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 06230e08714a24222d993f2f016169a6 2024-11-24T07:35:42,958 INFO [StoreOpener-06230e08714a24222d993f2f016169a6-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 06230e08714a24222d993f2f016169a6 columnFamilyName a 2024-11-24T07:35:42,958 DEBUG [StoreOpener-06230e08714a24222d993f2f016169a6-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T07:35:42,959 INFO [StoreOpener-06230e08714a24222d993f2f016169a6-1 {}] regionserver.HStore(327): Store=06230e08714a24222d993f2f016169a6/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T07:35:42,959 INFO [StoreOpener-06230e08714a24222d993f2f016169a6-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 06230e08714a24222d993f2f016169a6 2024-11-24T07:35:42,960 INFO [StoreOpener-06230e08714a24222d993f2f016169a6-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 06230e08714a24222d993f2f016169a6 columnFamilyName b 2024-11-24T07:35:42,960 DEBUG [StoreOpener-06230e08714a24222d993f2f016169a6-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T07:35:42,960 INFO [StoreOpener-06230e08714a24222d993f2f016169a6-1 {}] regionserver.HStore(327): Store=06230e08714a24222d993f2f016169a6/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T07:35:42,961 INFO [StoreOpener-06230e08714a24222d993f2f016169a6-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 06230e08714a24222d993f2f016169a6 2024-11-24T07:35:42,961 INFO [StoreOpener-06230e08714a24222d993f2f016169a6-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 06230e08714a24222d993f2f016169a6 columnFamilyName c 2024-11-24T07:35:42,962 DEBUG [StoreOpener-06230e08714a24222d993f2f016169a6-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T07:35:42,962 INFO [StoreOpener-06230e08714a24222d993f2f016169a6-1 {}] regionserver.HStore(327): Store=06230e08714a24222d993f2f016169a6/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T07:35:42,962 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 06230e08714a24222d993f2f016169a6 2024-11-24T07:35:42,963 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33505/hbase/data/default/testReplayEditsAfterAbortingFlush/06230e08714a24222d993f2f016169a6 2024-11-24T07:35:42,964 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:33505/hbase/data/default/testReplayEditsAfterAbortingFlush/06230e08714a24222d993f2f016169a6 2024-11-24T07:35:42,965 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:33505/hbase/data/default/testReplayEditsAfterAbortingFlush/06230e08714a24222d993f2f016169a6/recovered.edits/0000000000000000026 2024-11-24T07:35:42,967 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:33505/hbase/data/default/testReplayEditsAfterAbortingFlush/06230e08714a24222d993f2f016169a6/recovered.edits/0000000000000000026: isRecoveredEdits=true, hasTagCompression=true, hasValueCompression=true, valueCompressionType=GZ 2024-11-24T07:35:42,969 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 20, skipped 0, firstSequenceIdInLog=4, maxSequenceIdInLog=26, path=hdfs://localhost:33505/hbase/data/default/testReplayEditsAfterAbortingFlush/06230e08714a24222d993f2f016169a6/recovered.edits/0000000000000000026 2024-11-24T07:35:42,969 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 06230e08714a24222d993f2f016169a6 3/3 column families, dataSize=1.16 KB heapSize=3.41 KB 2024-11-24T07:35:42,985 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33505/hbase/data/default/testReplayEditsAfterAbortingFlush/06230e08714a24222d993f2f016169a6/.tmp/a/b58393cf404b42e48c9300b15f1b9f53 is 64, key is testReplayEditsAfterAbortingFlush12/a:q/1732433742886/Put/seqid=0 2024-11-24T07:35:42,993 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42685 is added to blk_1073741911_1089 (size=5523) 2024-11-24T07:35:42,994 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41971 is added to blk_1073741911_1089 (size=5523) 2024-11-24T07:35:42,994 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41161 is added to blk_1073741911_1089 (size=5523) 2024-11-24T07:35:42,994 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=416 B at sequenceid=26 (bloomFilter=true), to=hdfs://localhost:33505/hbase/data/default/testReplayEditsAfterAbortingFlush/06230e08714a24222d993f2f016169a6/.tmp/a/b58393cf404b42e48c9300b15f1b9f53 2024-11-24T07:35:43,016 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33505/hbase/data/default/testReplayEditsAfterAbortingFlush/06230e08714a24222d993f2f016169a6/.tmp/b/97f4500451e948fdafd65462cc438756 is 64, key is testReplayEditsAfterAbortingFlush10/b:q/1732433742878/Put/seqid=0 2024-11-24T07:35:43,024 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42685 is added to blk_1073741912_1090 (size=5524) 2024-11-24T07:35:43,024 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41971 is added to blk_1073741912_1090 (size=5524) 2024-11-24T07:35:43,025 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41161 is added to blk_1073741912_1090 (size=5524) 2024-11-24T07:35:43,025 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=417 B at sequenceid=26 (bloomFilter=true), to=hdfs://localhost:33505/hbase/data/default/testReplayEditsAfterAbortingFlush/06230e08714a24222d993f2f016169a6/.tmp/b/97f4500451e948fdafd65462cc438756 2024-11-24T07:35:43,045 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33505/hbase/data/default/testReplayEditsAfterAbortingFlush/06230e08714a24222d993f2f016169a6/.tmp/c/e119e48616de45798476d741fe197e2b is 64, key is testReplayEditsAfterAbortingFlush11/c:q/1732433742881/Put/seqid=0 2024-11-24T07:35:43,050 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testReplayEditsWrittenViaHRegion 2024-11-24T07:35:43,051 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testReplayEditsWrittenViaHRegion Metrics about Tables on a single HBase RegionServer 2024-11-24T07:35:43,051 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42685 is added to blk_1073741913_1091 (size=5457) 2024-11-24T07:35:43,052 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41971 is added to blk_1073741913_1091 (size=5457) 2024-11-24T07:35:43,052 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41161 is added to blk_1073741913_1091 (size=5457) 2024-11-24T07:35:43,052 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=357 B at sequenceid=26 (bloomFilter=true), to=hdfs://localhost:33505/hbase/data/default/testReplayEditsAfterAbortingFlush/06230e08714a24222d993f2f016169a6/.tmp/c/e119e48616de45798476d741fe197e2b 2024-11-24T07:35:43,058 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33505/hbase/data/default/testReplayEditsAfterAbortingFlush/06230e08714a24222d993f2f016169a6/.tmp/a/b58393cf404b42e48c9300b15f1b9f53 as hdfs://localhost:33505/hbase/data/default/testReplayEditsAfterAbortingFlush/06230e08714a24222d993f2f016169a6/a/b58393cf404b42e48c9300b15f1b9f53 2024-11-24T07:35:43,064 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33505/hbase/data/default/testReplayEditsAfterAbortingFlush/06230e08714a24222d993f2f016169a6/a/b58393cf404b42e48c9300b15f1b9f53, entries=7, sequenceid=26, filesize=5.4 K 2024-11-24T07:35:43,065 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33505/hbase/data/default/testReplayEditsAfterAbortingFlush/06230e08714a24222d993f2f016169a6/.tmp/b/97f4500451e948fdafd65462cc438756 as hdfs://localhost:33505/hbase/data/default/testReplayEditsAfterAbortingFlush/06230e08714a24222d993f2f016169a6/b/97f4500451e948fdafd65462cc438756 2024-11-24T07:35:43,070 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33505/hbase/data/default/testReplayEditsAfterAbortingFlush/06230e08714a24222d993f2f016169a6/b/97f4500451e948fdafd65462cc438756, entries=7, sequenceid=26, filesize=5.4 K 2024-11-24T07:35:43,071 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33505/hbase/data/default/testReplayEditsAfterAbortingFlush/06230e08714a24222d993f2f016169a6/.tmp/c/e119e48616de45798476d741fe197e2b as hdfs://localhost:33505/hbase/data/default/testReplayEditsAfterAbortingFlush/06230e08714a24222d993f2f016169a6/c/e119e48616de45798476d741fe197e2b 2024-11-24T07:35:43,078 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33505/hbase/data/default/testReplayEditsAfterAbortingFlush/06230e08714a24222d993f2f016169a6/c/e119e48616de45798476d741fe197e2b, entries=6, sequenceid=26, filesize=5.3 K 2024-11-24T07:35:43,078 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~1.16 KB/1190, heapSize ~3.36 KB/3440, currentSize=0 B/0 for 06230e08714a24222d993f2f016169a6 in 109ms, sequenceid=26, compaction requested=false; wal=null 2024-11-24T07:35:43,079 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:33505/hbase/data/default/testReplayEditsAfterAbortingFlush/06230e08714a24222d993f2f016169a6/recovered.edits/0000000000000000026 2024-11-24T07:35:43,080 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 06230e08714a24222d993f2f016169a6 2024-11-24T07:35:43,080 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 06230e08714a24222d993f2f016169a6 2024-11-24T07:35:43,081 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsAfterAbortingFlush descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-24T07:35:43,082 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 06230e08714a24222d993f2f016169a6 2024-11-24T07:35:43,085 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:33505/hbase/data/default/testReplayEditsAfterAbortingFlush/06230e08714a24222d993f2f016169a6/recovered.edits/26.seqid, newMaxSeqId=26, maxSeqId=1 2024-11-24T07:35:43,086 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 06230e08714a24222d993f2f016169a6; next sequenceid=27; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=73842202, jitterRate=0.10033455491065979}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-24T07:35:43,086 DEBUG [Time-limited test {}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 06230e08714a24222d993f2f016169a6 2024-11-24T07:35:43,087 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 06230e08714a24222d993f2f016169a6: Running coprocessor pre-open hook at 1732433742955Writing region info on filesystem at 1732433742955Initializing all the Stores at 1732433742956 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732433742956Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732433742957 (+1 ms)Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732433742957Obtaining lock to block concurrent updates at 1732433742969 (+12 ms)Preparing flush snapshotting stores in 06230e08714a24222d993f2f016169a6 at 1732433742969Finished memstore snapshotting testReplayEditsAfterAbortingFlush,,1732433733273.06230e08714a24222d993f2f016169a6., syncing WAL and waiting on mvcc, flushsize=dataSize=1190, getHeapSize=3440, getOffHeapSize=0, getCellsCount=20 at 1732433742970 (+1 ms)Flushing stores of testReplayEditsAfterAbortingFlush,,1732433733273.06230e08714a24222d993f2f016169a6. at 1732433742970Flushing 06230e08714a24222d993f2f016169a6/a: creating writer at 1732433742970Flushing 06230e08714a24222d993f2f016169a6/a: appending metadata at 1732433742985 (+15 ms)Flushing 06230e08714a24222d993f2f016169a6/a: closing flushed file at 1732433742985Flushing 06230e08714a24222d993f2f016169a6/b: creating writer at 1732433743001 (+16 ms)Flushing 06230e08714a24222d993f2f016169a6/b: appending metadata at 1732433743016 (+15 ms)Flushing 06230e08714a24222d993f2f016169a6/b: closing flushed file at 1732433743016Flushing 06230e08714a24222d993f2f016169a6/c: creating writer at 1732433743030 (+14 ms)Flushing 06230e08714a24222d993f2f016169a6/c: appending metadata at 1732433743045 (+15 ms)Flushing 06230e08714a24222d993f2f016169a6/c: closing flushed file at 1732433743045Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@594c293b: reopening flushed file at 1732433743057 (+12 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@464cfa6b: reopening flushed file at 1732433743064 (+7 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@282fc303: reopening flushed file at 1732433743070 (+6 ms)Finished flush of dataSize ~1.16 KB/1190, heapSize ~3.36 KB/3440, currentSize=0 B/0 for 06230e08714a24222d993f2f016169a6 in 109ms, sequenceid=26, compaction requested=false; wal=null at 1732433743078 (+8 ms)Cleaning up temporary data from old regions at 1732433743080 (+2 ms)Running coprocessor post-open hooks at 1732433743086 (+6 ms)Region opened successfully at 1732433743087 (+1 ms) 2024-11-24T07:35:43,109 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestAsyncWALReplayValueCompression#testReplayEditsAfterAbortingFlush Thread=418 (was 422), OpenFileDescriptor=1182 (was 1126) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=184 (was 217), ProcessCount=11 (was 11), AvailableMemoryMB=2002 (was 2091) 2024-11-24T07:35:43,109 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1182 is superior to 1024 2024-11-24T07:35:43,122 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestAsyncWALReplayValueCompression#testDatalossWhenInputError Thread=418, OpenFileDescriptor=1182, MaxFileDescriptor=1048576, SystemLoadAverage=184, ProcessCount=11, AvailableMemoryMB=2001 2024-11-24T07:35:43,122 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1182 is superior to 1024 2024-11-24T07:35:43,141 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-24T07:35:43,143 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-24T07:35:43,144 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-24T07:35:43,146 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-31156165, suffix=, logDir=hdfs://localhost:33505/hbase/WALs/hregion-31156165, archiveDir=hdfs://localhost:33505/hbase/oldWALs, maxLogs=32 2024-11-24T07:35:43,166 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-31156165/hregion-31156165.1732433743147, exclude list is [], retry=0 2024-11-24T07:35:43,169 DEBUG [AsyncFSWAL-24-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41161,DS-b0260eac-1380-4e84-bf22-a60109fdf5f6,DISK] 2024-11-24T07:35:43,169 DEBUG [AsyncFSWAL-24-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41971,DS-a67a12dc-5f72-4d11-ac5a-3e225e33c62a,DISK] 2024-11-24T07:35:43,169 DEBUG [AsyncFSWAL-24-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:42685,DS-467ee9a1-f3ca-4279-9b61-69a2a6a0308f,DISK] 2024-11-24T07:35:43,171 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-31156165/hregion-31156165.1732433743147 2024-11-24T07:35:43,171 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:40647:40647),(127.0.0.1/127.0.0.1:37111:37111),(127.0.0.1/127.0.0.1:35537:35537)] 2024-11-24T07:35:43,172 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => 405fa8c79c3c8baf3514195084dc329f, NAME => 'testDatalossWhenInputError,,1732433743142.405fa8c79c3c8baf3514195084dc329f.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testDatalossWhenInputError', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:33505/hbase 2024-11-24T07:35:43,192 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42685 is added to blk_1073741915_1093 (size=61) 2024-11-24T07:35:43,192 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41161 is added to blk_1073741915_1093 (size=61) 2024-11-24T07:35:43,194 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41971 is added to blk_1073741915_1093 (size=61) 2024-11-24T07:35:43,194 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testDatalossWhenInputError,,1732433743142.405fa8c79c3c8baf3514195084dc329f.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T07:35:43,196 INFO [StoreOpener-405fa8c79c3c8baf3514195084dc329f-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 405fa8c79c3c8baf3514195084dc329f 2024-11-24T07:35:43,198 INFO [StoreOpener-405fa8c79c3c8baf3514195084dc329f-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 405fa8c79c3c8baf3514195084dc329f columnFamilyName a 2024-11-24T07:35:43,198 DEBUG [StoreOpener-405fa8c79c3c8baf3514195084dc329f-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T07:35:43,199 INFO [StoreOpener-405fa8c79c3c8baf3514195084dc329f-1 {}] regionserver.HStore(327): Store=405fa8c79c3c8baf3514195084dc329f/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T07:35:43,199 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 405fa8c79c3c8baf3514195084dc329f 2024-11-24T07:35:43,200 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33505/hbase/data/default/testDatalossWhenInputError/405fa8c79c3c8baf3514195084dc329f 2024-11-24T07:35:43,200 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33505/hbase/data/default/testDatalossWhenInputError/405fa8c79c3c8baf3514195084dc329f 2024-11-24T07:35:43,200 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 405fa8c79c3c8baf3514195084dc329f 2024-11-24T07:35:43,201 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 405fa8c79c3c8baf3514195084dc329f 2024-11-24T07:35:43,202 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 405fa8c79c3c8baf3514195084dc329f 2024-11-24T07:35:43,205 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:33505/hbase/data/default/testDatalossWhenInputError/405fa8c79c3c8baf3514195084dc329f/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-24T07:35:43,205 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 405fa8c79c3c8baf3514195084dc329f; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=65751343, jitterRate=-0.020228639245033264}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-24T07:35:43,206 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 405fa8c79c3c8baf3514195084dc329f: Writing region info on filesystem at 1732433743194Initializing all the Stores at 1732433743195 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732433743195Cleaning up temporary data from old regions at 1732433743201 (+6 ms)Region opened successfully at 1732433743206 (+5 ms) 2024-11-24T07:35:43,206 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 405fa8c79c3c8baf3514195084dc329f, disabling compactions & flushes 2024-11-24T07:35:43,206 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testDatalossWhenInputError,,1732433743142.405fa8c79c3c8baf3514195084dc329f. 2024-11-24T07:35:43,206 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testDatalossWhenInputError,,1732433743142.405fa8c79c3c8baf3514195084dc329f. 2024-11-24T07:35:43,206 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testDatalossWhenInputError,,1732433743142.405fa8c79c3c8baf3514195084dc329f. after waiting 0 ms 2024-11-24T07:35:43,206 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testDatalossWhenInputError,,1732433743142.405fa8c79c3c8baf3514195084dc329f. 2024-11-24T07:35:43,207 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testDatalossWhenInputError,,1732433743142.405fa8c79c3c8baf3514195084dc329f. 2024-11-24T07:35:43,207 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 405fa8c79c3c8baf3514195084dc329f: Waiting for close lock at 1732433743206Disabling compacts and flushes for region at 1732433743206Disabling writes for close at 1732433743206Writing region close event to WAL at 1732433743207 (+1 ms)Closed at 1732433743207 2024-11-24T07:35:43,210 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42685 is added to blk_1073741914_1092 (size=95) 2024-11-24T07:35:43,210 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41161 is added to blk_1073741914_1092 (size=95) 2024-11-24T07:35:43,210 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41971 is added to blk_1073741914_1092 (size=95) 2024-11-24T07:35:43,215 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-11-24T07:35:43,215 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-31156165:(num 1732433743147) 2024-11-24T07:35:43,215 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-24T07:35:43,218 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:33505/hbase/WALs/testdatalosswheninputerror-manual,16010,1732433743141, archiveDir=hdfs://localhost:33505/hbase/oldWALs, maxLogs=32 2024-11-24T07:35:43,238 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testdatalosswheninputerror-manual,16010,1732433743141/wal.1732433743218, exclude list is [], retry=0 2024-11-24T07:35:43,241 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41971,DS-a67a12dc-5f72-4d11-ac5a-3e225e33c62a,DISK] 2024-11-24T07:35:43,242 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41161,DS-b0260eac-1380-4e84-bf22-a60109fdf5f6,DISK] 2024-11-24T07:35:43,242 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:42685,DS-467ee9a1-f3ca-4279-9b61-69a2a6a0308f,DISK] 2024-11-24T07:35:43,244 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testdatalosswheninputerror-manual,16010,1732433743141/wal.1732433743218 2024-11-24T07:35:43,244 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:37111:37111),(127.0.0.1/127.0.0.1:40647:40647),(127.0.0.1/127.0.0.1:35537:35537)] 2024-11-24T07:35:43,244 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 405fa8c79c3c8baf3514195084dc329f, NAME => 'testDatalossWhenInputError,,1732433743142.405fa8c79c3c8baf3514195084dc329f.', STARTKEY => '', ENDKEY => ''} 2024-11-24T07:35:43,244 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testDatalossWhenInputError,,1732433743142.405fa8c79c3c8baf3514195084dc329f.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T07:35:43,244 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 405fa8c79c3c8baf3514195084dc329f 2024-11-24T07:35:43,244 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 405fa8c79c3c8baf3514195084dc329f 2024-11-24T07:35:43,246 INFO [StoreOpener-405fa8c79c3c8baf3514195084dc329f-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 405fa8c79c3c8baf3514195084dc329f 2024-11-24T07:35:43,247 INFO [StoreOpener-405fa8c79c3c8baf3514195084dc329f-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 405fa8c79c3c8baf3514195084dc329f columnFamilyName a 2024-11-24T07:35:43,247 DEBUG [StoreOpener-405fa8c79c3c8baf3514195084dc329f-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T07:35:43,248 INFO [StoreOpener-405fa8c79c3c8baf3514195084dc329f-1 {}] regionserver.HStore(327): Store=405fa8c79c3c8baf3514195084dc329f/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T07:35:43,248 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 405fa8c79c3c8baf3514195084dc329f 2024-11-24T07:35:43,248 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33505/hbase/data/default/testDatalossWhenInputError/405fa8c79c3c8baf3514195084dc329f 2024-11-24T07:35:43,250 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33505/hbase/data/default/testDatalossWhenInputError/405fa8c79c3c8baf3514195084dc329f 2024-11-24T07:35:43,250 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 405fa8c79c3c8baf3514195084dc329f 2024-11-24T07:35:43,250 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 405fa8c79c3c8baf3514195084dc329f 2024-11-24T07:35:43,252 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 405fa8c79c3c8baf3514195084dc329f 2024-11-24T07:35:43,253 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 405fa8c79c3c8baf3514195084dc329f; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=61341829, jitterRate=-0.08593551814556122}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-24T07:35:43,253 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 405fa8c79c3c8baf3514195084dc329f: Writing region info on filesystem at 1732433743245Initializing all the Stores at 1732433743246 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732433743246Cleaning up temporary data from old regions at 1732433743250 (+4 ms)Region opened successfully at 1732433743253 (+3 ms) 2024-11-24T07:35:43,262 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 405fa8c79c3c8baf3514195084dc329f, disabling compactions & flushes 2024-11-24T07:35:43,263 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testDatalossWhenInputError,,1732433743142.405fa8c79c3c8baf3514195084dc329f. 2024-11-24T07:35:43,263 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testDatalossWhenInputError,,1732433743142.405fa8c79c3c8baf3514195084dc329f. 2024-11-24T07:35:43,263 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testDatalossWhenInputError,,1732433743142.405fa8c79c3c8baf3514195084dc329f. after waiting 0 ms 2024-11-24T07:35:43,263 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testDatalossWhenInputError,,1732433743142.405fa8c79c3c8baf3514195084dc329f. 2024-11-24T07:35:43,263 ERROR [Time-limited test {}] regionserver.HRegion(1960): Memstore data size is 750 in region testDatalossWhenInputError,,1732433743142.405fa8c79c3c8baf3514195084dc329f. 2024-11-24T07:35:43,263 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testDatalossWhenInputError,,1732433743142.405fa8c79c3c8baf3514195084dc329f. 2024-11-24T07:35:43,263 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 405fa8c79c3c8baf3514195084dc329f: Waiting for close lock at 1732433743262Disabling compacts and flushes for region at 1732433743262Disabling writes for close at 1732433743263 (+1 ms)Writing region close event to WAL at 1732433743263Closed at 1732433743263 2024-11-24T07:35:43,268 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42685 is added to blk_1073741916_1094 (size=1050) 2024-11-24T07:35:43,268 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41971 is added to blk_1073741916_1094 (size=1050) 2024-11-24T07:35:43,269 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41161 is added to blk_1073741916_1094 (size=1050) 2024-11-24T07:35:43,284 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:33505/hbase/WALs/testdatalosswheninputerror-manual,16010,1732433743141/wal.1732433743218, size=1.0 K (1050bytes) 2024-11-24T07:35:43,284 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:33505/hbase/WALs/testdatalosswheninputerror-manual,16010,1732433743141/wal.1732433743218 2024-11-24T07:35:43,285 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:33505/hbase/WALs/testdatalosswheninputerror-manual,16010,1732433743141/wal.1732433743218 after 1ms 2024-11-24T07:35:43,287 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:33505/hbase/WALs/testdatalosswheninputerror-manual,16010,1732433743141/wal.1732433743218: isRecoveredEdits=false, hasTagCompression=true, hasValueCompression=true, valueCompressionType=GZ 2024-11-24T07:35:43,287 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:33505/hbase/WALs/testdatalosswheninputerror-manual,16010,1732433743141/wal.1732433743218 took 3ms 2024-11-24T07:35:43,289 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:33505/hbase/WALs/testdatalosswheninputerror-manual,16010,1732433743141/wal.1732433743218 so closing down 2024-11-24T07:35:43,289 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-24T07:35:43,290 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000003-wal.1732433743218.temp 2024-11-24T07:35:43,291 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:33505/hbase/data/default/testDatalossWhenInputError/405fa8c79c3c8baf3514195084dc329f/recovered.edits/0000000000000000003-wal.1732433743218.temp 2024-11-24T07:35:43,291 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-11-24T07:35:43,319 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42685 is added to blk_1073741917_1095 (size=1050) 2024-11-24T07:35:43,319 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41161 is added to blk_1073741917_1095 (size=1050) 2024-11-24T07:35:43,320 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41971 is added to blk_1073741917_1095 (size=1050) 2024-11-24T07:35:43,320 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:33505/hbase/data/default/testDatalossWhenInputError/405fa8c79c3c8baf3514195084dc329f/recovered.edits/0000000000000000003-wal.1732433743218.temp (wrote 10 edits, skipped 0 edits in 0 ms) 2024-11-24T07:35:43,321 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:33505/hbase/data/default/testDatalossWhenInputError/405fa8c79c3c8baf3514195084dc329f/recovered.edits/0000000000000000003-wal.1732433743218.temp to hdfs://localhost:33505/hbase/data/default/testDatalossWhenInputError/405fa8c79c3c8baf3514195084dc329f/recovered.edits/0000000000000000012 2024-11-24T07:35:43,322 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 10 edits across 1 Regions in 35 ms; skipped=0; WAL=hdfs://localhost:33505/hbase/WALs/testdatalosswheninputerror-manual,16010,1732433743141/wal.1732433743218, size=1.0 K, length=1050, corrupted=false, cancelled=false 2024-11-24T07:35:43,322 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:33505/hbase/WALs/testdatalosswheninputerror-manual,16010,1732433743141/wal.1732433743218, journal: Splitting hdfs://localhost:33505/hbase/WALs/testdatalosswheninputerror-manual,16010,1732433743141/wal.1732433743218, size=1.0 K (1050bytes) at 1732433743284Finishing writing output for hdfs://localhost:33505/hbase/WALs/testdatalosswheninputerror-manual,16010,1732433743141/wal.1732433743218 so closing down at 1732433743289 (+5 ms)Creating recovered edits writer path=hdfs://localhost:33505/hbase/data/default/testDatalossWhenInputError/405fa8c79c3c8baf3514195084dc329f/recovered.edits/0000000000000000003-wal.1732433743218.temp at 1732433743291 (+2 ms)3 split writer threads finished at 1732433743291Closed recovered edits writer path=hdfs://localhost:33505/hbase/data/default/testDatalossWhenInputError/405fa8c79c3c8baf3514195084dc329f/recovered.edits/0000000000000000003-wal.1732433743218.temp (wrote 10 edits, skipped 0 edits in 0 ms) at 1732433743320 (+29 ms)Rename recovered edits hdfs://localhost:33505/hbase/data/default/testDatalossWhenInputError/405fa8c79c3c8baf3514195084dc329f/recovered.edits/0000000000000000003-wal.1732433743218.temp to hdfs://localhost:33505/hbase/data/default/testDatalossWhenInputError/405fa8c79c3c8baf3514195084dc329f/recovered.edits/0000000000000000012 at 1732433743321 (+1 ms)Processed 10 edits across 1 Regions in 35 ms; skipped=0; WAL=hdfs://localhost:33505/hbase/WALs/testdatalosswheninputerror-manual,16010,1732433743141/wal.1732433743218, size=1.0 K, length=1050, corrupted=false, cancelled=false at 1732433743322 (+1 ms) 2024-11-24T07:35:43,323 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:33505/hbase/WALs/testdatalosswheninputerror-manual,16010,1732433743141/wal.1732433743218 to hdfs://localhost:33505/hbase/oldWALs/wal.1732433743218 2024-11-24T07:35:43,324 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:33505/hbase/data/default/testDatalossWhenInputError/405fa8c79c3c8baf3514195084dc329f/recovered.edits/0000000000000000012 2024-11-24T07:35:43,327 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:33505/hbase/data/default/testDatalossWhenInputError/405fa8c79c3c8baf3514195084dc329f/recovered.edits/0000000000000000012: isRecoveredEdits=true, hasTagCompression=true, hasValueCompression=true, valueCompressionType=GZ 2024-11-24T07:35:43,639 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-24T07:35:43,641 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:33505/hbase/WALs/testdatalosswheninputerror-manual,16010,1732433743141, archiveDir=hdfs://localhost:33505/hbase/oldWALs, maxLogs=32 2024-11-24T07:35:43,654 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testdatalosswheninputerror-manual,16010,1732433743141/wal.1732433743641, exclude list is [], retry=0 2024-11-24T07:35:43,657 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:42685,DS-467ee9a1-f3ca-4279-9b61-69a2a6a0308f,DISK] 2024-11-24T07:35:43,657 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41161,DS-b0260eac-1380-4e84-bf22-a60109fdf5f6,DISK] 2024-11-24T07:35:43,657 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41971,DS-a67a12dc-5f72-4d11-ac5a-3e225e33c62a,DISK] 2024-11-24T07:35:43,659 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testdatalosswheninputerror-manual,16010,1732433743141/wal.1732433743641 2024-11-24T07:35:43,659 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:35537:35537),(127.0.0.1/127.0.0.1:40647:40647),(127.0.0.1/127.0.0.1:37111:37111)] 2024-11-24T07:35:43,659 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 405fa8c79c3c8baf3514195084dc329f, NAME => 'testDatalossWhenInputError,,1732433743142.405fa8c79c3c8baf3514195084dc329f.', STARTKEY => '', ENDKEY => ''} 2024-11-24T07:35:43,659 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testDatalossWhenInputError,,1732433743142.405fa8c79c3c8baf3514195084dc329f.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T07:35:43,659 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 405fa8c79c3c8baf3514195084dc329f 2024-11-24T07:35:43,659 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 405fa8c79c3c8baf3514195084dc329f 2024-11-24T07:35:43,662 INFO [StoreOpener-405fa8c79c3c8baf3514195084dc329f-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 405fa8c79c3c8baf3514195084dc329f 2024-11-24T07:35:43,664 INFO [StoreOpener-405fa8c79c3c8baf3514195084dc329f-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 405fa8c79c3c8baf3514195084dc329f columnFamilyName a 2024-11-24T07:35:43,664 DEBUG [StoreOpener-405fa8c79c3c8baf3514195084dc329f-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T07:35:43,665 INFO [StoreOpener-405fa8c79c3c8baf3514195084dc329f-1 {}] regionserver.HStore(327): Store=405fa8c79c3c8baf3514195084dc329f/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T07:35:43,665 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 405fa8c79c3c8baf3514195084dc329f 2024-11-24T07:35:43,666 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33505/hbase/data/default/testDatalossWhenInputError/405fa8c79c3c8baf3514195084dc329f 2024-11-24T07:35:43,667 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:33505/hbase/data/default/testDatalossWhenInputError/405fa8c79c3c8baf3514195084dc329f 2024-11-24T07:35:43,667 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:33505/hbase/data/default/testDatalossWhenInputError/405fa8c79c3c8baf3514195084dc329f/recovered.edits/0000000000000000012 2024-11-24T07:35:43,669 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:33505/hbase/data/default/testDatalossWhenInputError/405fa8c79c3c8baf3514195084dc329f/recovered.edits/0000000000000000012: isRecoveredEdits=true, hasTagCompression=true, hasValueCompression=true, valueCompressionType=GZ 2024-11-24T07:35:43,670 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 10, skipped 0, firstSequenceIdInLog=3, maxSequenceIdInLog=12, path=hdfs://localhost:33505/hbase/data/default/testDatalossWhenInputError/405fa8c79c3c8baf3514195084dc329f/recovered.edits/0000000000000000012 2024-11-24T07:35:43,670 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 405fa8c79c3c8baf3514195084dc329f 1/1 column families, dataSize=750 B heapSize=1.73 KB 2024-11-24T07:35:43,689 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33505/hbase/data/default/testDatalossWhenInputError/405fa8c79c3c8baf3514195084dc329f/.tmp/a/9c6b26e14b4b43aba043e5052ca4f991 is 79, key is testDatalossWhenInputError/a:x0/1732433743254/Put/seqid=0 2024-11-24T07:35:43,694 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41161 is added to blk_1073741919_1097 (size=5808) 2024-11-24T07:35:43,694 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42685 is added to blk_1073741919_1097 (size=5808) 2024-11-24T07:35:43,695 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41971 is added to blk_1073741919_1097 (size=5808) 2024-11-24T07:35:43,695 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=750 B at sequenceid=12 (bloomFilter=true), to=hdfs://localhost:33505/hbase/data/default/testDatalossWhenInputError/405fa8c79c3c8baf3514195084dc329f/.tmp/a/9c6b26e14b4b43aba043e5052ca4f991 2024-11-24T07:35:43,704 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33505/hbase/data/default/testDatalossWhenInputError/405fa8c79c3c8baf3514195084dc329f/.tmp/a/9c6b26e14b4b43aba043e5052ca4f991 as hdfs://localhost:33505/hbase/data/default/testDatalossWhenInputError/405fa8c79c3c8baf3514195084dc329f/a/9c6b26e14b4b43aba043e5052ca4f991 2024-11-24T07:35:43,714 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33505/hbase/data/default/testDatalossWhenInputError/405fa8c79c3c8baf3514195084dc329f/a/9c6b26e14b4b43aba043e5052ca4f991, entries=10, sequenceid=12, filesize=5.7 K 2024-11-24T07:35:43,714 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~750 B/750, heapSize ~1.72 KB/1760, currentSize=0 B/0 for 405fa8c79c3c8baf3514195084dc329f in 44ms, sequenceid=12, compaction requested=false; wal=null 2024-11-24T07:35:43,715 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:33505/hbase/data/default/testDatalossWhenInputError/405fa8c79c3c8baf3514195084dc329f/recovered.edits/0000000000000000012 2024-11-24T07:35:43,716 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 405fa8c79c3c8baf3514195084dc329f 2024-11-24T07:35:43,716 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 405fa8c79c3c8baf3514195084dc329f 2024-11-24T07:35:43,719 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 405fa8c79c3c8baf3514195084dc329f 2024-11-24T07:35:43,721 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:33505/hbase/data/default/testDatalossWhenInputError/405fa8c79c3c8baf3514195084dc329f/recovered.edits/12.seqid, newMaxSeqId=12, maxSeqId=1 2024-11-24T07:35:43,722 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 405fa8c79c3c8baf3514195084dc329f; next sequenceid=13; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=60556948, jitterRate=-0.09763115644454956}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-24T07:35:43,722 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 405fa8c79c3c8baf3514195084dc329f: Writing region info on filesystem at 1732433743659Initializing all the Stores at 1732433743662 (+3 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732433743662Obtaining lock to block concurrent updates at 1732433743671 (+9 ms)Preparing flush snapshotting stores in 405fa8c79c3c8baf3514195084dc329f at 1732433743671Finished memstore snapshotting testDatalossWhenInputError,,1732433743142.405fa8c79c3c8baf3514195084dc329f., syncing WAL and waiting on mvcc, flushsize=dataSize=750, getHeapSize=1760, getOffHeapSize=0, getCellsCount=10 at 1732433743671Flushing stores of testDatalossWhenInputError,,1732433743142.405fa8c79c3c8baf3514195084dc329f. at 1732433743671Flushing 405fa8c79c3c8baf3514195084dc329f/a: creating writer at 1732433743671Flushing 405fa8c79c3c8baf3514195084dc329f/a: appending metadata at 1732433743688 (+17 ms)Flushing 405fa8c79c3c8baf3514195084dc329f/a: closing flushed file at 1732433743688Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@3edfd371: reopening flushed file at 1732433743702 (+14 ms)Finished flush of dataSize ~750 B/750, heapSize ~1.72 KB/1760, currentSize=0 B/0 for 405fa8c79c3c8baf3514195084dc329f in 44ms, sequenceid=12, compaction requested=false; wal=null at 1732433743714 (+12 ms)Cleaning up temporary data from old regions at 1732433743716 (+2 ms)Region opened successfully at 1732433743722 (+6 ms) 2024-11-24T07:35:43,725 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 405fa8c79c3c8baf3514195084dc329f, NAME => 'testDatalossWhenInputError,,1732433743142.405fa8c79c3c8baf3514195084dc329f.', STARTKEY => '', ENDKEY => ''} 2024-11-24T07:35:43,726 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testDatalossWhenInputError,,1732433743142.405fa8c79c3c8baf3514195084dc329f.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T07:35:43,726 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 405fa8c79c3c8baf3514195084dc329f 2024-11-24T07:35:43,726 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 405fa8c79c3c8baf3514195084dc329f 2024-11-24T07:35:43,727 INFO [StoreOpener-405fa8c79c3c8baf3514195084dc329f-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 405fa8c79c3c8baf3514195084dc329f 2024-11-24T07:35:43,728 INFO [StoreOpener-405fa8c79c3c8baf3514195084dc329f-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 405fa8c79c3c8baf3514195084dc329f columnFamilyName a 2024-11-24T07:35:43,728 DEBUG [StoreOpener-405fa8c79c3c8baf3514195084dc329f-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T07:35:43,732 DEBUG [StoreOpener-405fa8c79c3c8baf3514195084dc329f-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:33505/hbase/data/default/testDatalossWhenInputError/405fa8c79c3c8baf3514195084dc329f/a/9c6b26e14b4b43aba043e5052ca4f991 2024-11-24T07:35:43,732 INFO [StoreOpener-405fa8c79c3c8baf3514195084dc329f-1 {}] regionserver.HStore(327): Store=405fa8c79c3c8baf3514195084dc329f/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T07:35:43,732 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 405fa8c79c3c8baf3514195084dc329f 2024-11-24T07:35:43,733 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33505/hbase/data/default/testDatalossWhenInputError/405fa8c79c3c8baf3514195084dc329f 2024-11-24T07:35:43,734 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33505/hbase/data/default/testDatalossWhenInputError/405fa8c79c3c8baf3514195084dc329f 2024-11-24T07:35:43,734 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 405fa8c79c3c8baf3514195084dc329f 2024-11-24T07:35:43,734 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 405fa8c79c3c8baf3514195084dc329f 2024-11-24T07:35:43,735 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 405fa8c79c3c8baf3514195084dc329f 2024-11-24T07:35:43,737 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:33505/hbase/data/default/testDatalossWhenInputError/405fa8c79c3c8baf3514195084dc329f/recovered.edits/13.seqid, newMaxSeqId=13, maxSeqId=12 2024-11-24T07:35:43,738 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 405fa8c79c3c8baf3514195084dc329f; next sequenceid=14; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=65461347, jitterRate=-0.02454991638660431}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-24T07:35:43,738 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 405fa8c79c3c8baf3514195084dc329f: Writing region info on filesystem at 1732433743726Initializing all the Stores at 1732433743727 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732433743727Cleaning up temporary data from old regions at 1732433743734 (+7 ms)Region opened successfully at 1732433743738 (+4 ms) 2024-11-24T07:35:43,754 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestAsyncWALReplayValueCompression#testDatalossWhenInputError Thread=428 (was 418) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-166876350_22 at /127.0.0.1:39328 [Receiving block BP-1337235608-172.17.0.3-1732433708251:blk_1073741918_1096] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-1337235608-172.17.0.3-1732433708251:blk_1073741918_1096, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-24-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-1337235608-172.17.0.3-1732433708251:blk_1073741918_1096, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-1337235608-172.17.0.3-1732433708251:blk_1073741918_1096, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-24-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-166876350_22 at /127.0.0.1:57406 [Receiving block BP-1337235608-172.17.0.3-1732433708251:blk_1073741918_1096] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-166876350_22 at /127.0.0.1:51050 [Waiting for operation #11] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-166876350_22 at /127.0.0.1:57320 [Waiting for operation #23] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-166876350_22 at /127.0.0.1:39280 [Waiting for operation #8] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-166876350_22 at /127.0.0.1:51134 [Receiving block BP-1337235608-172.17.0.3-1732433708251:blk_1073741918_1096] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-24-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=1264 (was 1182) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=184 (was 184), ProcessCount=11 (was 11), AvailableMemoryMB=1924 (was 2001) 2024-11-24T07:35:43,754 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1264 is superior to 1024 2024-11-24T07:35:43,765 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestAsyncWALReplayValueCompression#testCompactedBulkLoadedFiles Thread=428, OpenFileDescriptor=1264, MaxFileDescriptor=1048576, SystemLoadAverage=184, ProcessCount=11, AvailableMemoryMB=1924 2024-11-24T07:35:43,765 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1264 is superior to 1024 2024-11-24T07:35:43,779 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-24T07:35:43,781 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-24T07:35:43,781 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-24T07:35:43,783 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-19542996, suffix=, logDir=hdfs://localhost:33505/hbase/WALs/hregion-19542996, archiveDir=hdfs://localhost:33505/hbase/oldWALs, maxLogs=32 2024-11-24T07:35:43,794 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-19542996/hregion-19542996.1732433743783, exclude list is [], retry=0 2024-11-24T07:35:43,797 DEBUG [AsyncFSWAL-26-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:42685,DS-467ee9a1-f3ca-4279-9b61-69a2a6a0308f,DISK] 2024-11-24T07:35:43,797 DEBUG [AsyncFSWAL-26-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41971,DS-a67a12dc-5f72-4d11-ac5a-3e225e33c62a,DISK] 2024-11-24T07:35:43,797 DEBUG [AsyncFSWAL-26-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41161,DS-b0260eac-1380-4e84-bf22-a60109fdf5f6,DISK] 2024-11-24T07:35:43,800 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-19542996/hregion-19542996.1732433743783 2024-11-24T07:35:43,800 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:35537:35537),(127.0.0.1/127.0.0.1:37111:37111),(127.0.0.1/127.0.0.1:40647:40647)] 2024-11-24T07:35:43,800 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => c56c8f7a1b18dc97cf10423d456ae9a8, NAME => 'testCompactedBulkLoadedFiles,,1732433743780.c56c8f7a1b18dc97cf10423d456ae9a8.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testCompactedBulkLoadedFiles', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:33505/hbase 2024-11-24T07:35:43,809 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42685 is added to blk_1073741921_1099 (size=63) 2024-11-24T07:35:43,810 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41161 is added to blk_1073741921_1099 (size=63) 2024-11-24T07:35:43,810 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41971 is added to blk_1073741921_1099 (size=63) 2024-11-24T07:35:43,811 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testCompactedBulkLoadedFiles,,1732433743780.c56c8f7a1b18dc97cf10423d456ae9a8.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T07:35:43,813 INFO [StoreOpener-c56c8f7a1b18dc97cf10423d456ae9a8-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region c56c8f7a1b18dc97cf10423d456ae9a8 2024-11-24T07:35:43,814 INFO [StoreOpener-c56c8f7a1b18dc97cf10423d456ae9a8-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region c56c8f7a1b18dc97cf10423d456ae9a8 columnFamilyName a 2024-11-24T07:35:43,814 DEBUG [StoreOpener-c56c8f7a1b18dc97cf10423d456ae9a8-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T07:35:43,814 INFO [StoreOpener-c56c8f7a1b18dc97cf10423d456ae9a8-1 {}] regionserver.HStore(327): Store=c56c8f7a1b18dc97cf10423d456ae9a8/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T07:35:43,815 INFO [StoreOpener-c56c8f7a1b18dc97cf10423d456ae9a8-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region c56c8f7a1b18dc97cf10423d456ae9a8 2024-11-24T07:35:43,816 INFO [StoreOpener-c56c8f7a1b18dc97cf10423d456ae9a8-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region c56c8f7a1b18dc97cf10423d456ae9a8 columnFamilyName b 2024-11-24T07:35:43,816 DEBUG [StoreOpener-c56c8f7a1b18dc97cf10423d456ae9a8-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T07:35:43,816 INFO [StoreOpener-c56c8f7a1b18dc97cf10423d456ae9a8-1 {}] regionserver.HStore(327): Store=c56c8f7a1b18dc97cf10423d456ae9a8/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T07:35:43,816 INFO [StoreOpener-c56c8f7a1b18dc97cf10423d456ae9a8-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region c56c8f7a1b18dc97cf10423d456ae9a8 2024-11-24T07:35:43,818 INFO [StoreOpener-c56c8f7a1b18dc97cf10423d456ae9a8-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region c56c8f7a1b18dc97cf10423d456ae9a8 columnFamilyName c 2024-11-24T07:35:43,818 DEBUG [StoreOpener-c56c8f7a1b18dc97cf10423d456ae9a8-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T07:35:43,818 INFO [StoreOpener-c56c8f7a1b18dc97cf10423d456ae9a8-1 {}] regionserver.HStore(327): Store=c56c8f7a1b18dc97cf10423d456ae9a8/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T07:35:43,818 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for c56c8f7a1b18dc97cf10423d456ae9a8 2024-11-24T07:35:43,819 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33505/hbase/data/default/testCompactedBulkLoadedFiles/c56c8f7a1b18dc97cf10423d456ae9a8 2024-11-24T07:35:43,820 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33505/hbase/data/default/testCompactedBulkLoadedFiles/c56c8f7a1b18dc97cf10423d456ae9a8 2024-11-24T07:35:43,821 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for c56c8f7a1b18dc97cf10423d456ae9a8 2024-11-24T07:35:43,821 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for c56c8f7a1b18dc97cf10423d456ae9a8 2024-11-24T07:35:43,821 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testCompactedBulkLoadedFiles descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-24T07:35:43,823 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for c56c8f7a1b18dc97cf10423d456ae9a8 2024-11-24T07:35:43,825 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:33505/hbase/data/default/testCompactedBulkLoadedFiles/c56c8f7a1b18dc97cf10423d456ae9a8/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-24T07:35:43,826 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened c56c8f7a1b18dc97cf10423d456ae9a8; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=71579328, jitterRate=0.06661510467529297}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-24T07:35:43,826 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for c56c8f7a1b18dc97cf10423d456ae9a8: Writing region info on filesystem at 1732433743811Initializing all the Stores at 1732433743811Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732433743811Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732433743813 (+2 ms)Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732433743813Cleaning up temporary data from old regions at 1732433743821 (+8 ms)Region opened successfully at 1732433743826 (+5 ms) 2024-11-24T07:35:43,826 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing c56c8f7a1b18dc97cf10423d456ae9a8, disabling compactions & flushes 2024-11-24T07:35:43,826 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testCompactedBulkLoadedFiles,,1732433743780.c56c8f7a1b18dc97cf10423d456ae9a8. 2024-11-24T07:35:43,826 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testCompactedBulkLoadedFiles,,1732433743780.c56c8f7a1b18dc97cf10423d456ae9a8. 2024-11-24T07:35:43,826 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testCompactedBulkLoadedFiles,,1732433743780.c56c8f7a1b18dc97cf10423d456ae9a8. after waiting 0 ms 2024-11-24T07:35:43,826 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testCompactedBulkLoadedFiles,,1732433743780.c56c8f7a1b18dc97cf10423d456ae9a8. 2024-11-24T07:35:43,827 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testCompactedBulkLoadedFiles,,1732433743780.c56c8f7a1b18dc97cf10423d456ae9a8. 2024-11-24T07:35:43,827 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for c56c8f7a1b18dc97cf10423d456ae9a8: Waiting for close lock at 1732433743826Disabling compacts and flushes for region at 1732433743826Disabling writes for close at 1732433743826Writing region close event to WAL at 1732433743827 (+1 ms)Closed at 1732433743827 2024-11-24T07:35:43,829 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42685 is added to blk_1073741920_1098 (size=95) 2024-11-24T07:35:43,830 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41971 is added to blk_1073741920_1098 (size=95) 2024-11-24T07:35:43,830 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41161 is added to blk_1073741920_1098 (size=95) 2024-11-24T07:35:43,831 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-11-24T07:35:43,832 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-19542996:(num 1732433743783) 2024-11-24T07:35:43,832 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-24T07:35:43,833 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:33505/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1732433743778, archiveDir=hdfs://localhost:33505/hbase/oldWALs, maxLogs=32 2024-11-24T07:35:43,846 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1732433743778/wal.1732433743833, exclude list is [], retry=0 2024-11-24T07:35:43,848 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:42685,DS-467ee9a1-f3ca-4279-9b61-69a2a6a0308f,DISK] 2024-11-24T07:35:43,849 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41161,DS-b0260eac-1380-4e84-bf22-a60109fdf5f6,DISK] 2024-11-24T07:35:43,849 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41971,DS-a67a12dc-5f72-4d11-ac5a-3e225e33c62a,DISK] 2024-11-24T07:35:43,850 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1732433743778/wal.1732433743833 2024-11-24T07:35:43,850 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:35537:35537),(127.0.0.1/127.0.0.1:40647:40647),(127.0.0.1/127.0.0.1:37111:37111)] 2024-11-24T07:35:43,851 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => c56c8f7a1b18dc97cf10423d456ae9a8, NAME => 'testCompactedBulkLoadedFiles,,1732433743780.c56c8f7a1b18dc97cf10423d456ae9a8.', STARTKEY => '', ENDKEY => ''} 2024-11-24T07:35:43,851 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testCompactedBulkLoadedFiles,,1732433743780.c56c8f7a1b18dc97cf10423d456ae9a8.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T07:35:43,851 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for c56c8f7a1b18dc97cf10423d456ae9a8 2024-11-24T07:35:43,851 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for c56c8f7a1b18dc97cf10423d456ae9a8 2024-11-24T07:35:43,852 INFO [StoreOpener-c56c8f7a1b18dc97cf10423d456ae9a8-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region c56c8f7a1b18dc97cf10423d456ae9a8 2024-11-24T07:35:43,853 INFO [StoreOpener-c56c8f7a1b18dc97cf10423d456ae9a8-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region c56c8f7a1b18dc97cf10423d456ae9a8 columnFamilyName a 2024-11-24T07:35:43,853 DEBUG [StoreOpener-c56c8f7a1b18dc97cf10423d456ae9a8-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T07:35:43,854 INFO [StoreOpener-c56c8f7a1b18dc97cf10423d456ae9a8-1 {}] regionserver.HStore(327): Store=c56c8f7a1b18dc97cf10423d456ae9a8/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T07:35:43,854 INFO [StoreOpener-c56c8f7a1b18dc97cf10423d456ae9a8-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region c56c8f7a1b18dc97cf10423d456ae9a8 2024-11-24T07:35:43,855 INFO [StoreOpener-c56c8f7a1b18dc97cf10423d456ae9a8-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region c56c8f7a1b18dc97cf10423d456ae9a8 columnFamilyName b 2024-11-24T07:35:43,855 DEBUG [StoreOpener-c56c8f7a1b18dc97cf10423d456ae9a8-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T07:35:43,855 INFO [StoreOpener-c56c8f7a1b18dc97cf10423d456ae9a8-1 {}] regionserver.HStore(327): Store=c56c8f7a1b18dc97cf10423d456ae9a8/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T07:35:43,855 INFO [StoreOpener-c56c8f7a1b18dc97cf10423d456ae9a8-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region c56c8f7a1b18dc97cf10423d456ae9a8 2024-11-24T07:35:43,856 INFO [StoreOpener-c56c8f7a1b18dc97cf10423d456ae9a8-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region c56c8f7a1b18dc97cf10423d456ae9a8 columnFamilyName c 2024-11-24T07:35:43,856 DEBUG [StoreOpener-c56c8f7a1b18dc97cf10423d456ae9a8-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T07:35:43,856 INFO [StoreOpener-c56c8f7a1b18dc97cf10423d456ae9a8-1 {}] regionserver.HStore(327): Store=c56c8f7a1b18dc97cf10423d456ae9a8/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T07:35:43,857 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for c56c8f7a1b18dc97cf10423d456ae9a8 2024-11-24T07:35:43,857 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33505/hbase/data/default/testCompactedBulkLoadedFiles/c56c8f7a1b18dc97cf10423d456ae9a8 2024-11-24T07:35:43,858 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33505/hbase/data/default/testCompactedBulkLoadedFiles/c56c8f7a1b18dc97cf10423d456ae9a8 2024-11-24T07:35:43,859 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for c56c8f7a1b18dc97cf10423d456ae9a8 2024-11-24T07:35:43,859 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for c56c8f7a1b18dc97cf10423d456ae9a8 2024-11-24T07:35:43,859 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testCompactedBulkLoadedFiles descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-24T07:35:43,861 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for c56c8f7a1b18dc97cf10423d456ae9a8 2024-11-24T07:35:43,861 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened c56c8f7a1b18dc97cf10423d456ae9a8; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=66557429, jitterRate=-0.008217021822929382}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-24T07:35:43,862 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for c56c8f7a1b18dc97cf10423d456ae9a8: Writing region info on filesystem at 1732433743851Initializing all the Stores at 1732433743852 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732433743852Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732433743852Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732433743852Cleaning up temporary data from old regions at 1732433743859 (+7 ms)Region opened successfully at 1732433743862 (+3 ms) 2024-11-24T07:35:43,866 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33505/hbase/testCompactedBulkLoadedFiles/hfile0 is 32, key is 000/a:a/1732433743865/Put/seqid=0 2024-11-24T07:35:43,871 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42685 is added to blk_1073741923_1101 (size=4875) 2024-11-24T07:35:43,872 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41161 is added to blk_1073741923_1101 (size=4875) 2024-11-24T07:35:43,872 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41971 is added to blk_1073741923_1101 (size=4875) 2024-11-24T07:35:43,874 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33505/hbase/testCompactedBulkLoadedFiles/hfile1 is 32, key is 100/a:a/1732433743874/Put/seqid=0 2024-11-24T07:35:43,882 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41971 is added to blk_1073741924_1102 (size=4875) 2024-11-24T07:35:43,882 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42685 is added to blk_1073741924_1102 (size=4875) 2024-11-24T07:35:43,882 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41161 is added to blk_1073741924_1102 (size=4875) 2024-11-24T07:35:43,885 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33505/hbase/testCompactedBulkLoadedFiles/hfile2 is 32, key is 200/a:a/1732433743884/Put/seqid=0 2024-11-24T07:35:43,891 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41161 is added to blk_1073741925_1103 (size=4875) 2024-11-24T07:35:43,891 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41971 is added to blk_1073741925_1103 (size=4875) 2024-11-24T07:35:43,891 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42685 is added to blk_1073741925_1103 (size=4875) 2024-11-24T07:35:43,892 INFO [Time-limited test {}] regionserver.HStore(614): Validating hfile at hdfs://localhost:33505/hbase/testCompactedBulkLoadedFiles/hfile0 for inclusion in c56c8f7a1b18dc97cf10423d456ae9a8/a 2024-11-24T07:35:43,895 DEBUG [Time-limited test {}] regionserver.HStore(626): HFile bounds: first=000 last=050 2024-11-24T07:35:43,896 DEBUG [Time-limited test {}] regionserver.HStore(628): Region bounds: first= last= 2024-11-24T07:35:43,896 INFO [Time-limited test {}] regionserver.HStore(614): Validating hfile at hdfs://localhost:33505/hbase/testCompactedBulkLoadedFiles/hfile1 for inclusion in c56c8f7a1b18dc97cf10423d456ae9a8/a 2024-11-24T07:35:43,900 DEBUG [Time-limited test {}] regionserver.HStore(626): HFile bounds: first=100 last=150 2024-11-24T07:35:43,900 DEBUG [Time-limited test {}] regionserver.HStore(628): Region bounds: first= last= 2024-11-24T07:35:43,900 INFO [Time-limited test {}] regionserver.HStore(614): Validating hfile at hdfs://localhost:33505/hbase/testCompactedBulkLoadedFiles/hfile2 for inclusion in c56c8f7a1b18dc97cf10423d456ae9a8/a 2024-11-24T07:35:43,904 DEBUG [Time-limited test {}] regionserver.HStore(626): HFile bounds: first=200 last=250 2024-11-24T07:35:43,904 DEBUG [Time-limited test {}] regionserver.HStore(628): Region bounds: first= last= 2024-11-24T07:35:43,904 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing c56c8f7a1b18dc97cf10423d456ae9a8 3/3 column families, dataSize=51 B heapSize=896 B 2024-11-24T07:35:43,922 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33505/hbase/data/default/testCompactedBulkLoadedFiles/c56c8f7a1b18dc97cf10423d456ae9a8/.tmp/a/645cf1a37882435bbec84e89a890d4a3 is 55, key is testCompactedBulkLoadedFiles/a:a/1732433743862/Put/seqid=0 2024-11-24T07:35:43,928 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41161 is added to blk_1073741926_1104 (size=5107) 2024-11-24T07:35:43,928 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41971 is added to blk_1073741926_1104 (size=5107) 2024-11-24T07:35:43,929 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42685 is added to blk_1073741926_1104 (size=5107) 2024-11-24T07:35:43,929 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51 B at sequenceid=4 (bloomFilter=true), to=hdfs://localhost:33505/hbase/data/default/testCompactedBulkLoadedFiles/c56c8f7a1b18dc97cf10423d456ae9a8/.tmp/a/645cf1a37882435bbec84e89a890d4a3 2024-11-24T07:35:43,934 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33505/hbase/data/default/testCompactedBulkLoadedFiles/c56c8f7a1b18dc97cf10423d456ae9a8/.tmp/a/645cf1a37882435bbec84e89a890d4a3 as hdfs://localhost:33505/hbase/data/default/testCompactedBulkLoadedFiles/c56c8f7a1b18dc97cf10423d456ae9a8/a/645cf1a37882435bbec84e89a890d4a3 2024-11-24T07:35:43,940 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33505/hbase/data/default/testCompactedBulkLoadedFiles/c56c8f7a1b18dc97cf10423d456ae9a8/a/645cf1a37882435bbec84e89a890d4a3, entries=1, sequenceid=4, filesize=5.0 K 2024-11-24T07:35:43,942 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~51 B/51, heapSize ~368 B/368, currentSize=0 B/0 for c56c8f7a1b18dc97cf10423d456ae9a8 in 37ms, sequenceid=4, compaction requested=false 2024-11-24T07:35:43,942 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for c56c8f7a1b18dc97cf10423d456ae9a8: 2024-11-24T07:35:43,943 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33505/hbase/testCompactedBulkLoadedFiles/hfile0 as hdfs://localhost:33505/hbase/data/default/testCompactedBulkLoadedFiles/c56c8f7a1b18dc97cf10423d456ae9a8/a/b351d40e64d04caca6572c30df830c03_SeqId_4_ 2024-11-24T07:35:43,944 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33505/hbase/testCompactedBulkLoadedFiles/hfile1 as hdfs://localhost:33505/hbase/data/default/testCompactedBulkLoadedFiles/c56c8f7a1b18dc97cf10423d456ae9a8/a/ee2db7a7071e4db68b7b9e22fb820050_SeqId_4_ 2024-11-24T07:35:43,945 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33505/hbase/testCompactedBulkLoadedFiles/hfile2 as hdfs://localhost:33505/hbase/data/default/testCompactedBulkLoadedFiles/c56c8f7a1b18dc97cf10423d456ae9a8/a/486e8609adac4ec7809b8056ecf172a0_SeqId_4_ 2024-11-24T07:35:43,946 INFO [Time-limited test {}] regionserver.HStore(700): Loaded HFile hdfs://localhost:33505/hbase/testCompactedBulkLoadedFiles/hfile0 into c56c8f7a1b18dc97cf10423d456ae9a8/a as hdfs://localhost:33505/hbase/data/default/testCompactedBulkLoadedFiles/c56c8f7a1b18dc97cf10423d456ae9a8/a/b351d40e64d04caca6572c30df830c03_SeqId_4_ - updating store file list. 2024-11-24T07:35:43,951 DEBUG [Time-limited test {}] regionserver.HStoreFile(483): HFile Bloom filter type for b351d40e64d04caca6572c30df830c03_SeqId_4_: NONE, but ROW specified in column family configuration 2024-11-24T07:35:43,951 INFO [Time-limited test {}] regionserver.HStore(722): Loaded HFile hdfs://localhost:33505/hbase/data/default/testCompactedBulkLoadedFiles/c56c8f7a1b18dc97cf10423d456ae9a8/a/b351d40e64d04caca6572c30df830c03_SeqId_4_ into c56c8f7a1b18dc97cf10423d456ae9a8/a 2024-11-24T07:35:43,951 INFO [Time-limited test {}] regionserver.HStore(706): Successfully loaded hdfs://localhost:33505/hbase/testCompactedBulkLoadedFiles/hfile0 into c56c8f7a1b18dc97cf10423d456ae9a8/a (new location: hdfs://localhost:33505/hbase/data/default/testCompactedBulkLoadedFiles/c56c8f7a1b18dc97cf10423d456ae9a8/a/b351d40e64d04caca6572c30df830c03_SeqId_4_) 2024-11-24T07:35:43,952 INFO [Time-limited test {}] regionserver.HStore(700): Loaded HFile hdfs://localhost:33505/hbase/testCompactedBulkLoadedFiles/hfile1 into c56c8f7a1b18dc97cf10423d456ae9a8/a as hdfs://localhost:33505/hbase/data/default/testCompactedBulkLoadedFiles/c56c8f7a1b18dc97cf10423d456ae9a8/a/ee2db7a7071e4db68b7b9e22fb820050_SeqId_4_ - updating store file list. 2024-11-24T07:35:43,956 DEBUG [Time-limited test {}] regionserver.HStoreFile(483): HFile Bloom filter type for ee2db7a7071e4db68b7b9e22fb820050_SeqId_4_: NONE, but ROW specified in column family configuration 2024-11-24T07:35:43,956 INFO [Time-limited test {}] regionserver.HStore(722): Loaded HFile hdfs://localhost:33505/hbase/data/default/testCompactedBulkLoadedFiles/c56c8f7a1b18dc97cf10423d456ae9a8/a/ee2db7a7071e4db68b7b9e22fb820050_SeqId_4_ into c56c8f7a1b18dc97cf10423d456ae9a8/a 2024-11-24T07:35:43,957 INFO [Time-limited test {}] regionserver.HStore(706): Successfully loaded hdfs://localhost:33505/hbase/testCompactedBulkLoadedFiles/hfile1 into c56c8f7a1b18dc97cf10423d456ae9a8/a (new location: hdfs://localhost:33505/hbase/data/default/testCompactedBulkLoadedFiles/c56c8f7a1b18dc97cf10423d456ae9a8/a/ee2db7a7071e4db68b7b9e22fb820050_SeqId_4_) 2024-11-24T07:35:43,958 INFO [Time-limited test {}] regionserver.HStore(700): Loaded HFile hdfs://localhost:33505/hbase/testCompactedBulkLoadedFiles/hfile2 into c56c8f7a1b18dc97cf10423d456ae9a8/a as hdfs://localhost:33505/hbase/data/default/testCompactedBulkLoadedFiles/c56c8f7a1b18dc97cf10423d456ae9a8/a/486e8609adac4ec7809b8056ecf172a0_SeqId_4_ - updating store file list. 2024-11-24T07:35:43,962 DEBUG [Time-limited test {}] regionserver.HStoreFile(483): HFile Bloom filter type for 486e8609adac4ec7809b8056ecf172a0_SeqId_4_: NONE, but ROW specified in column family configuration 2024-11-24T07:35:43,962 INFO [Time-limited test {}] regionserver.HStore(722): Loaded HFile hdfs://localhost:33505/hbase/data/default/testCompactedBulkLoadedFiles/c56c8f7a1b18dc97cf10423d456ae9a8/a/486e8609adac4ec7809b8056ecf172a0_SeqId_4_ into c56c8f7a1b18dc97cf10423d456ae9a8/a 2024-11-24T07:35:43,962 INFO [Time-limited test {}] regionserver.HStore(706): Successfully loaded hdfs://localhost:33505/hbase/testCompactedBulkLoadedFiles/hfile2 into c56c8f7a1b18dc97cf10423d456ae9a8/a (new location: hdfs://localhost:33505/hbase/data/default/testCompactedBulkLoadedFiles/c56c8f7a1b18dc97cf10423d456ae9a8/a/486e8609adac4ec7809b8056ecf172a0_SeqId_4_) 2024-11-24T07:35:43,970 DEBUG [Time-limited test {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-24T07:35:43,970 DEBUG [Time-limited test {}] regionserver.HStore(1541): c56c8f7a1b18dc97cf10423d456ae9a8/a is initiating major compaction (all files) 2024-11-24T07:35:43,970 INFO [Time-limited test {}] regionserver.HRegion(2416): Starting compaction of c56c8f7a1b18dc97cf10423d456ae9a8/a in testCompactedBulkLoadedFiles,,1732433743780.c56c8f7a1b18dc97cf10423d456ae9a8. 2024-11-24T07:35:43,970 INFO [Time-limited test {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33505/hbase/data/default/testCompactedBulkLoadedFiles/c56c8f7a1b18dc97cf10423d456ae9a8/a/645cf1a37882435bbec84e89a890d4a3, hdfs://localhost:33505/hbase/data/default/testCompactedBulkLoadedFiles/c56c8f7a1b18dc97cf10423d456ae9a8/a/b351d40e64d04caca6572c30df830c03_SeqId_4_, hdfs://localhost:33505/hbase/data/default/testCompactedBulkLoadedFiles/c56c8f7a1b18dc97cf10423d456ae9a8/a/ee2db7a7071e4db68b7b9e22fb820050_SeqId_4_, hdfs://localhost:33505/hbase/data/default/testCompactedBulkLoadedFiles/c56c8f7a1b18dc97cf10423d456ae9a8/a/486e8609adac4ec7809b8056ecf172a0_SeqId_4_] into tmpdir=hdfs://localhost:33505/hbase/data/default/testCompactedBulkLoadedFiles/c56c8f7a1b18dc97cf10423d456ae9a8/.tmp, totalSize=19.3 K 2024-11-24T07:35:43,971 DEBUG [Time-limited test {}] compactions.Compactor(225): Compacting 645cf1a37882435bbec84e89a890d4a3, keycount=1, bloomtype=ROW, size=5.0 K, encoding=NONE, compression=NONE, seqNum=4, earliestPutTs=1732433743862 2024-11-24T07:35:43,971 DEBUG [Time-limited test {}] compactions.Compactor(225): Compacting b351d40e64d04caca6572c30df830c03_SeqId_4_, keycount=10, bloomtype=NONE, size=4.8 K, encoding=NONE, compression=NONE, seqNum=4, earliestPutTs=-9223372036854775808 2024-11-24T07:35:43,971 DEBUG [Time-limited test {}] compactions.Compactor(225): Compacting ee2db7a7071e4db68b7b9e22fb820050_SeqId_4_, keycount=10, bloomtype=NONE, size=4.8 K, encoding=NONE, compression=NONE, seqNum=4, earliestPutTs=-9223372036854775808 2024-11-24T07:35:43,972 DEBUG [Time-limited test {}] compactions.Compactor(225): Compacting 486e8609adac4ec7809b8056ecf172a0_SeqId_4_, keycount=10, bloomtype=NONE, size=4.8 K, encoding=NONE, compression=NONE, seqNum=4, earliestPutTs=-9223372036854775808 2024-11-24T07:35:43,983 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33505/hbase/data/default/testCompactedBulkLoadedFiles/c56c8f7a1b18dc97cf10423d456ae9a8/.tmp/a/4a451deb33c24d1a9ae1f6fb76bd7a84 is 55, key is testCompactedBulkLoadedFiles/a:a/1732433743862/Put/seqid=0 2024-11-24T07:35:43,989 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41161 is added to blk_1073741927_1105 (size=6154) 2024-11-24T07:35:43,989 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41971 is added to blk_1073741927_1105 (size=6154) 2024-11-24T07:35:43,989 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42685 is added to blk_1073741927_1105 (size=6154) 2024-11-24T07:35:43,996 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33505/hbase/data/default/testCompactedBulkLoadedFiles/c56c8f7a1b18dc97cf10423d456ae9a8/.tmp/a/4a451deb33c24d1a9ae1f6fb76bd7a84 as hdfs://localhost:33505/hbase/data/default/testCompactedBulkLoadedFiles/c56c8f7a1b18dc97cf10423d456ae9a8/a/4a451deb33c24d1a9ae1f6fb76bd7a84 2024-11-24T07:35:44,003 INFO [Time-limited test {}] regionserver.HStore(1337): Completed major compaction of 4 (all) file(s) in c56c8f7a1b18dc97cf10423d456ae9a8/a of c56c8f7a1b18dc97cf10423d456ae9a8 into 4a451deb33c24d1a9ae1f6fb76bd7a84(size=6.0 K), total size for store is 6.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T07:35:44,003 DEBUG [Time-limited test {}] regionserver.HRegion(2446): Compaction status journal for c56c8f7a1b18dc97cf10423d456ae9a8: 2024-11-24T07:35:44,004 DEBUG [Time-limited test {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 0 store files, 0 compacting, 0 eligible, 16 blocking 2024-11-24T07:35:44,004 DEBUG [Time-limited test {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 0 store files, 0 compacting, 0 eligible, 16 blocking 2024-11-24T07:35:44,036 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:33505/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1732433743778/wal.1732433743833, size=0 (0bytes) 2024-11-24T07:35:44,036 WARN [Time-limited test {}] wal.WALSplitter(453): File hdfs://localhost:33505/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1732433743778/wal.1732433743833 might be still open, length is 0 2024-11-24T07:35:44,036 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:33505/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1732433743778/wal.1732433743833 2024-11-24T07:35:44,036 WARN [IPC Server handler 0 on default port 33505 {}] namenode.FSNamesystem(3854): DIR* NameSystem.internalReleaseLease: File /hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1732433743778/wal.1732433743833 has not been closed. Lease recovery is in progress. RecoveryId = 1106 for block blk_1073741922_1100 2024-11-24T07:35:44,037 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=0 on file=hdfs://localhost:33505/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1732433743778/wal.1732433743833 after 1ms 2024-11-24T07:35:44,854 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-166876350_22 at /127.0.0.1:57432 [Receiving block BP-1337235608-172.17.0.3-1732433708251:blk_1073741922_1100] {}] datanode.DataXceiver(331): 127.0.0.1:42685:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:57432 dst: /127.0.0.1:42685 java.io.InterruptedIOException: Interrupted while waiting for IO on channel java.nio.channels.SocketChannel[connected local=localhost/127.0.0.1:42685 remote=/127.0.0.1:57432]. Total timeout mills is 60000, 59149 millis timeout left. at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:350) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?] at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?] at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?] at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?] at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?] at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T07:35:44,855 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-166876350_22 at /127.0.0.1:39350 [Receiving block BP-1337235608-172.17.0.3-1732433708251:blk_1073741922_1100] {}] datanode.DataXceiver(331): 127.0.0.1:41971:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:39350 dst: /127.0.0.1:41971 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T07:35:44,855 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-166876350_22 at /127.0.0.1:51172 [Receiving block BP-1337235608-172.17.0.3-1732433708251:blk_1073741922_1100] {}] datanode.DataXceiver(331): 127.0.0.1:41161:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:51172 dst: /127.0.0.1:41161 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T07:35:44,859 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42685 is added to blk_1073741922_1106 (size=1174) 2024-11-24T07:35:44,860 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41161 is added to blk_1073741922_1106 (size=1174) 2024-11-24T07:35:44,860 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41971 is added to blk_1073741922_1106 (size=1174) 2024-11-24T07:35:48,038 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=1 on file=hdfs://localhost:33505/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1732433743778/wal.1732433743833 after 4002ms 2024-11-24T07:35:48,046 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:33505/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1732433743778/wal.1732433743833: isRecoveredEdits=false, hasTagCompression=true, hasValueCompression=true, valueCompressionType=GZ 2024-11-24T07:35:48,047 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:33505/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1732433743778/wal.1732433743833 took 4011ms 2024-11-24T07:35:48,049 INFO [Time-limited test {}] wal.WALSplitter(493): EOF from hdfs://localhost:33505/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1732433743778/wal.1732433743833; continuing. 2024-11-24T07:35:48,050 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:33505/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1732433743778/wal.1732433743833 so closing down 2024-11-24T07:35:48,050 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-24T07:35:48,051 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000003-wal.1732433743833.temp 2024-11-24T07:35:48,053 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:33505/hbase/data/default/testCompactedBulkLoadedFiles/c56c8f7a1b18dc97cf10423d456ae9a8/recovered.edits/0000000000000000003-wal.1732433743833.temp 2024-11-24T07:35:48,054 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-11-24T07:35:48,062 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41161 is added to blk_1073741928_1107 (size=548) 2024-11-24T07:35:48,062 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41971 is added to blk_1073741928_1107 (size=548) 2024-11-24T07:35:48,063 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42685 is added to blk_1073741928_1107 (size=548) 2024-11-24T07:35:48,063 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:33505/hbase/data/default/testCompactedBulkLoadedFiles/c56c8f7a1b18dc97cf10423d456ae9a8/recovered.edits/0000000000000000003-wal.1732433743833.temp (wrote 2 edits, skipped 0 edits in 0 ms) 2024-11-24T07:35:48,064 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:33505/hbase/data/default/testCompactedBulkLoadedFiles/c56c8f7a1b18dc97cf10423d456ae9a8/recovered.edits/0000000000000000003-wal.1732433743833.temp to hdfs://localhost:33505/hbase/data/default/testCompactedBulkLoadedFiles/c56c8f7a1b18dc97cf10423d456ae9a8/recovered.edits/0000000000000000008 2024-11-24T07:35:48,064 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 5 edits across 1 Regions in 17 ms; skipped=3; WAL=hdfs://localhost:33505/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1732433743778/wal.1732433743833, size=0, length=0, corrupted=false, cancelled=false 2024-11-24T07:35:48,064 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:33505/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1732433743778/wal.1732433743833, journal: Splitting hdfs://localhost:33505/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1732433743778/wal.1732433743833, size=0 (0bytes) at 1732433744036Finishing writing output for hdfs://localhost:33505/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1732433743778/wal.1732433743833 so closing down at 1732433748050 (+4014 ms)Creating recovered edits writer path=hdfs://localhost:33505/hbase/data/default/testCompactedBulkLoadedFiles/c56c8f7a1b18dc97cf10423d456ae9a8/recovered.edits/0000000000000000003-wal.1732433743833.temp at 1732433748054 (+4 ms)3 split writer threads finished at 1732433748054Closed recovered edits writer path=hdfs://localhost:33505/hbase/data/default/testCompactedBulkLoadedFiles/c56c8f7a1b18dc97cf10423d456ae9a8/recovered.edits/0000000000000000003-wal.1732433743833.temp (wrote 2 edits, skipped 0 edits in 0 ms) at 1732433748063 (+9 ms)Rename recovered edits hdfs://localhost:33505/hbase/data/default/testCompactedBulkLoadedFiles/c56c8f7a1b18dc97cf10423d456ae9a8/recovered.edits/0000000000000000003-wal.1732433743833.temp to hdfs://localhost:33505/hbase/data/default/testCompactedBulkLoadedFiles/c56c8f7a1b18dc97cf10423d456ae9a8/recovered.edits/0000000000000000008 at 1732433748064 (+1 ms)Processed 5 edits across 1 Regions in 17 ms; skipped=3; WAL=hdfs://localhost:33505/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1732433743778/wal.1732433743833, size=0, length=0, corrupted=false, cancelled=false at 1732433748064 2024-11-24T07:35:48,066 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:33505/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1732433743778/wal.1732433743833 to hdfs://localhost:33505/hbase/oldWALs/wal.1732433743833 2024-11-24T07:35:48,067 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:33505/hbase/data/default/testCompactedBulkLoadedFiles/c56c8f7a1b18dc97cf10423d456ae9a8/recovered.edits/0000000000000000008 2024-11-24T07:35:48,067 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-24T07:35:48,068 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:33505/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1732433743778, archiveDir=hdfs://localhost:33505/hbase/oldWALs, maxLogs=32 2024-11-24T07:35:48,080 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1732433743778/wal.1732433748068, exclude list is [], retry=0 2024-11-24T07:35:48,082 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:42685,DS-467ee9a1-f3ca-4279-9b61-69a2a6a0308f,DISK] 2024-11-24T07:35:48,083 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41161,DS-b0260eac-1380-4e84-bf22-a60109fdf5f6,DISK] 2024-11-24T07:35:48,083 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41971,DS-a67a12dc-5f72-4d11-ac5a-3e225e33c62a,DISK] 2024-11-24T07:35:48,085 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1732433743778/wal.1732433748068 2024-11-24T07:35:48,085 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:35537:35537),(127.0.0.1/127.0.0.1:40647:40647),(127.0.0.1/127.0.0.1:37111:37111)] 2024-11-24T07:35:48,085 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => c56c8f7a1b18dc97cf10423d456ae9a8, NAME => 'testCompactedBulkLoadedFiles,,1732433743780.c56c8f7a1b18dc97cf10423d456ae9a8.', STARTKEY => '', ENDKEY => ''} 2024-11-24T07:35:48,085 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testCompactedBulkLoadedFiles,,1732433743780.c56c8f7a1b18dc97cf10423d456ae9a8.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T07:35:48,085 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for c56c8f7a1b18dc97cf10423d456ae9a8 2024-11-24T07:35:48,086 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for c56c8f7a1b18dc97cf10423d456ae9a8 2024-11-24T07:35:48,087 INFO [StoreOpener-c56c8f7a1b18dc97cf10423d456ae9a8-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region c56c8f7a1b18dc97cf10423d456ae9a8 2024-11-24T07:35:48,088 INFO [StoreOpener-c56c8f7a1b18dc97cf10423d456ae9a8-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region c56c8f7a1b18dc97cf10423d456ae9a8 columnFamilyName a 2024-11-24T07:35:48,088 DEBUG [StoreOpener-c56c8f7a1b18dc97cf10423d456ae9a8-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T07:35:48,094 DEBUG [StoreFileOpener-c56c8f7a1b18dc97cf10423d456ae9a8-a-1 {}] regionserver.HStoreFile(483): HFile Bloom filter type for 486e8609adac4ec7809b8056ecf172a0_SeqId_4_: NONE, but ROW specified in column family configuration 2024-11-24T07:35:48,094 DEBUG [StoreOpener-c56c8f7a1b18dc97cf10423d456ae9a8-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:33505/hbase/data/default/testCompactedBulkLoadedFiles/c56c8f7a1b18dc97cf10423d456ae9a8/a/486e8609adac4ec7809b8056ecf172a0_SeqId_4_ 2024-11-24T07:35:48,099 DEBUG [StoreOpener-c56c8f7a1b18dc97cf10423d456ae9a8-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:33505/hbase/data/default/testCompactedBulkLoadedFiles/c56c8f7a1b18dc97cf10423d456ae9a8/a/4a451deb33c24d1a9ae1f6fb76bd7a84 2024-11-24T07:35:48,103 DEBUG [StoreOpener-c56c8f7a1b18dc97cf10423d456ae9a8-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:33505/hbase/data/default/testCompactedBulkLoadedFiles/c56c8f7a1b18dc97cf10423d456ae9a8/a/645cf1a37882435bbec84e89a890d4a3 2024-11-24T07:35:48,107 DEBUG [StoreFileOpener-c56c8f7a1b18dc97cf10423d456ae9a8-a-1 {}] regionserver.HStoreFile(483): HFile Bloom filter type for b351d40e64d04caca6572c30df830c03_SeqId_4_: NONE, but ROW specified in column family configuration 2024-11-24T07:35:48,107 DEBUG [StoreOpener-c56c8f7a1b18dc97cf10423d456ae9a8-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:33505/hbase/data/default/testCompactedBulkLoadedFiles/c56c8f7a1b18dc97cf10423d456ae9a8/a/b351d40e64d04caca6572c30df830c03_SeqId_4_ 2024-11-24T07:35:48,110 DEBUG [StoreFileOpener-c56c8f7a1b18dc97cf10423d456ae9a8-a-1 {}] regionserver.HStoreFile(483): HFile Bloom filter type for ee2db7a7071e4db68b7b9e22fb820050_SeqId_4_: NONE, but ROW specified in column family configuration 2024-11-24T07:35:48,110 DEBUG [StoreOpener-c56c8f7a1b18dc97cf10423d456ae9a8-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:33505/hbase/data/default/testCompactedBulkLoadedFiles/c56c8f7a1b18dc97cf10423d456ae9a8/a/ee2db7a7071e4db68b7b9e22fb820050_SeqId_4_ 2024-11-24T07:35:48,110 WARN [StoreOpener-c56c8f7a1b18dc97cf10423d456ae9a8-1 {}] regionserver.StoreEngine(317): Clearing the compacted storefile hdfs://localhost:33505/hbase/data/default/testCompactedBulkLoadedFiles/c56c8f7a1b18dc97cf10423d456ae9a8/a/486e8609adac4ec7809b8056ecf172a0_SeqId_4_ from org.apache.hadoop.hbase.regionserver.DefaultStoreEngine@39945d34 2024-11-24T07:35:48,111 WARN [StoreOpener-c56c8f7a1b18dc97cf10423d456ae9a8-1 {}] regionserver.StoreEngine(317): Clearing the compacted storefile hdfs://localhost:33505/hbase/data/default/testCompactedBulkLoadedFiles/c56c8f7a1b18dc97cf10423d456ae9a8/a/645cf1a37882435bbec84e89a890d4a3 from org.apache.hadoop.hbase.regionserver.DefaultStoreEngine@39945d34 2024-11-24T07:35:48,111 WARN [StoreOpener-c56c8f7a1b18dc97cf10423d456ae9a8-1 {}] regionserver.StoreEngine(317): Clearing the compacted storefile hdfs://localhost:33505/hbase/data/default/testCompactedBulkLoadedFiles/c56c8f7a1b18dc97cf10423d456ae9a8/a/b351d40e64d04caca6572c30df830c03_SeqId_4_ from org.apache.hadoop.hbase.regionserver.DefaultStoreEngine@39945d34 2024-11-24T07:35:48,111 WARN [StoreOpener-c56c8f7a1b18dc97cf10423d456ae9a8-1 {}] regionserver.StoreEngine(317): Clearing the compacted storefile hdfs://localhost:33505/hbase/data/default/testCompactedBulkLoadedFiles/c56c8f7a1b18dc97cf10423d456ae9a8/a/ee2db7a7071e4db68b7b9e22fb820050_SeqId_4_ from org.apache.hadoop.hbase.regionserver.DefaultStoreEngine@39945d34 2024-11-24T07:35:48,111 DEBUG [StoreOpener-c56c8f7a1b18dc97cf10423d456ae9a8-1 {}] regionserver.StoreEngine(327): Moving the files [hdfs://localhost:33505/hbase/data/default/testCompactedBulkLoadedFiles/c56c8f7a1b18dc97cf10423d456ae9a8/a/486e8609adac4ec7809b8056ecf172a0_SeqId_4_, hdfs://localhost:33505/hbase/data/default/testCompactedBulkLoadedFiles/c56c8f7a1b18dc97cf10423d456ae9a8/a/645cf1a37882435bbec84e89a890d4a3, hdfs://localhost:33505/hbase/data/default/testCompactedBulkLoadedFiles/c56c8f7a1b18dc97cf10423d456ae9a8/a/b351d40e64d04caca6572c30df830c03_SeqId_4_, hdfs://localhost:33505/hbase/data/default/testCompactedBulkLoadedFiles/c56c8f7a1b18dc97cf10423d456ae9a8/a/ee2db7a7071e4db68b7b9e22fb820050_SeqId_4_] to archive 2024-11-24T07:35:48,112 DEBUG [StoreOpener-c56c8f7a1b18dc97cf10423d456ae9a8-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-24T07:35:48,113 DEBUG [StoreOpener-c56c8f7a1b18dc97cf10423d456ae9a8-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33505/hbase/data/default/testCompactedBulkLoadedFiles/c56c8f7a1b18dc97cf10423d456ae9a8/a/486e8609adac4ec7809b8056ecf172a0_SeqId_4_ to hdfs://localhost:33505/hbase/archive/data/default/testCompactedBulkLoadedFiles/c56c8f7a1b18dc97cf10423d456ae9a8/a/486e8609adac4ec7809b8056ecf172a0_SeqId_4_ 2024-11-24T07:35:48,114 DEBUG [StoreOpener-c56c8f7a1b18dc97cf10423d456ae9a8-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33505/hbase/data/default/testCompactedBulkLoadedFiles/c56c8f7a1b18dc97cf10423d456ae9a8/a/645cf1a37882435bbec84e89a890d4a3 to hdfs://localhost:33505/hbase/archive/data/default/testCompactedBulkLoadedFiles/c56c8f7a1b18dc97cf10423d456ae9a8/a/645cf1a37882435bbec84e89a890d4a3 2024-11-24T07:35:48,116 DEBUG [StoreOpener-c56c8f7a1b18dc97cf10423d456ae9a8-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33505/hbase/data/default/testCompactedBulkLoadedFiles/c56c8f7a1b18dc97cf10423d456ae9a8/a/b351d40e64d04caca6572c30df830c03_SeqId_4_ to hdfs://localhost:33505/hbase/archive/data/default/testCompactedBulkLoadedFiles/c56c8f7a1b18dc97cf10423d456ae9a8/a/b351d40e64d04caca6572c30df830c03_SeqId_4_ 2024-11-24T07:35:48,117 DEBUG [StoreOpener-c56c8f7a1b18dc97cf10423d456ae9a8-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33505/hbase/data/default/testCompactedBulkLoadedFiles/c56c8f7a1b18dc97cf10423d456ae9a8/a/ee2db7a7071e4db68b7b9e22fb820050_SeqId_4_ to hdfs://localhost:33505/hbase/archive/data/default/testCompactedBulkLoadedFiles/c56c8f7a1b18dc97cf10423d456ae9a8/a/ee2db7a7071e4db68b7b9e22fb820050_SeqId_4_ 2024-11-24T07:35:48,117 INFO [StoreOpener-c56c8f7a1b18dc97cf10423d456ae9a8-1 {}] regionserver.HStore(327): Store=c56c8f7a1b18dc97cf10423d456ae9a8/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T07:35:48,117 INFO [StoreOpener-c56c8f7a1b18dc97cf10423d456ae9a8-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region c56c8f7a1b18dc97cf10423d456ae9a8 2024-11-24T07:35:48,118 INFO [StoreOpener-c56c8f7a1b18dc97cf10423d456ae9a8-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region c56c8f7a1b18dc97cf10423d456ae9a8 columnFamilyName b 2024-11-24T07:35:48,118 DEBUG [StoreOpener-c56c8f7a1b18dc97cf10423d456ae9a8-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T07:35:48,118 INFO [StoreOpener-c56c8f7a1b18dc97cf10423d456ae9a8-1 {}] regionserver.HStore(327): Store=c56c8f7a1b18dc97cf10423d456ae9a8/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T07:35:48,118 INFO [StoreOpener-c56c8f7a1b18dc97cf10423d456ae9a8-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region c56c8f7a1b18dc97cf10423d456ae9a8 2024-11-24T07:35:48,119 INFO [StoreOpener-c56c8f7a1b18dc97cf10423d456ae9a8-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region c56c8f7a1b18dc97cf10423d456ae9a8 columnFamilyName c 2024-11-24T07:35:48,119 DEBUG [StoreOpener-c56c8f7a1b18dc97cf10423d456ae9a8-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T07:35:48,119 INFO [StoreOpener-c56c8f7a1b18dc97cf10423d456ae9a8-1 {}] regionserver.HStore(327): Store=c56c8f7a1b18dc97cf10423d456ae9a8/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T07:35:48,119 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for c56c8f7a1b18dc97cf10423d456ae9a8 2024-11-24T07:35:48,120 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33505/hbase/data/default/testCompactedBulkLoadedFiles/c56c8f7a1b18dc97cf10423d456ae9a8 2024-11-24T07:35:48,121 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:33505/hbase/data/default/testCompactedBulkLoadedFiles/c56c8f7a1b18dc97cf10423d456ae9a8 2024-11-24T07:35:48,122 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:33505/hbase/data/default/testCompactedBulkLoadedFiles/c56c8f7a1b18dc97cf10423d456ae9a8/recovered.edits/0000000000000000008 2024-11-24T07:35:48,123 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:33505/hbase/data/default/testCompactedBulkLoadedFiles/c56c8f7a1b18dc97cf10423d456ae9a8/recovered.edits/0000000000000000008: isRecoveredEdits=true, hasTagCompression=true, hasValueCompression=true, valueCompressionType=GZ 2024-11-24T07:35:48,124 DEBUG [Time-limited test {}] regionserver.HRegion(5836): c56c8f7a1b18dc97cf10423d456ae9a8 : Replaying compaction marker table_name: "testCompactedBulkLoadedFiles" encoded_region_name: "c56c8f7a1b18dc97cf10423d456ae9a8" family_name: "a" compaction_input: "645cf1a37882435bbec84e89a890d4a3" compaction_input: "b351d40e64d04caca6572c30df830c03_SeqId_4_" compaction_input: "ee2db7a7071e4db68b7b9e22fb820050_SeqId_4_" compaction_input: "486e8609adac4ec7809b8056ecf172a0_SeqId_4_" compaction_output: "4a451deb33c24d1a9ae1f6fb76bd7a84" store_home_dir: "a" region_name: "testCompactedBulkLoadedFiles,,1732433743780.c56c8f7a1b18dc97cf10423d456ae9a8." with seqId=9223372036854775807 and lastReplayedOpenRegionSeqId=-1 2024-11-24T07:35:48,124 DEBUG [Time-limited test {}] regionserver.HStore(1354): Completing compaction from the WAL marker 2024-11-24T07:35:48,125 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 0, skipped 2, firstSequenceIdInLog=3, maxSequenceIdInLog=8, path=hdfs://localhost:33505/hbase/data/default/testCompactedBulkLoadedFiles/c56c8f7a1b18dc97cf10423d456ae9a8/recovered.edits/0000000000000000008 2024-11-24T07:35:48,125 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:33505/hbase/data/default/testCompactedBulkLoadedFiles/c56c8f7a1b18dc97cf10423d456ae9a8/recovered.edits/0000000000000000008 2024-11-24T07:35:48,126 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for c56c8f7a1b18dc97cf10423d456ae9a8 2024-11-24T07:35:48,126 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for c56c8f7a1b18dc97cf10423d456ae9a8 2024-11-24T07:35:48,127 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testCompactedBulkLoadedFiles descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-24T07:35:48,128 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for c56c8f7a1b18dc97cf10423d456ae9a8 2024-11-24T07:35:48,130 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:33505/hbase/data/default/testCompactedBulkLoadedFiles/c56c8f7a1b18dc97cf10423d456ae9a8/recovered.edits/8.seqid, newMaxSeqId=8, maxSeqId=1 2024-11-24T07:35:48,131 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened c56c8f7a1b18dc97cf10423d456ae9a8; next sequenceid=9; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=67349939, jitterRate=0.0035922974348068237}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-24T07:35:48,131 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for c56c8f7a1b18dc97cf10423d456ae9a8: Writing region info on filesystem at 1732433748086Initializing all the Stores at 1732433748086Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732433748087 (+1 ms)Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732433748087Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732433748087Cleaning up temporary data from old regions at 1732433748126 (+39 ms)Region opened successfully at 1732433748131 (+5 ms) 2024-11-24T07:35:48,134 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing c56c8f7a1b18dc97cf10423d456ae9a8, disabling compactions & flushes 2024-11-24T07:35:48,134 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testCompactedBulkLoadedFiles,,1732433743780.c56c8f7a1b18dc97cf10423d456ae9a8. 2024-11-24T07:35:48,134 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testCompactedBulkLoadedFiles,,1732433743780.c56c8f7a1b18dc97cf10423d456ae9a8. 2024-11-24T07:35:48,134 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testCompactedBulkLoadedFiles,,1732433743780.c56c8f7a1b18dc97cf10423d456ae9a8. after waiting 0 ms 2024-11-24T07:35:48,134 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testCompactedBulkLoadedFiles,,1732433743780.c56c8f7a1b18dc97cf10423d456ae9a8. 2024-11-24T07:35:48,134 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testCompactedBulkLoadedFiles,,1732433743780.c56c8f7a1b18dc97cf10423d456ae9a8. 2024-11-24T07:35:48,134 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for c56c8f7a1b18dc97cf10423d456ae9a8: Waiting for close lock at 1732433748133Disabling compacts and flushes for region at 1732433748133Disabling writes for close at 1732433748134 (+1 ms)Writing region close event to WAL at 1732433748134Closed at 1732433748134 2024-11-24T07:35:48,137 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42685 is added to blk_1073741929_1108 (size=95) 2024-11-24T07:35:48,137 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41971 is added to blk_1073741929_1108 (size=95) 2024-11-24T07:35:48,137 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41161 is added to blk_1073741929_1108 (size=95) 2024-11-24T07:35:48,139 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-11-24T07:35:48,139 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL wal:(num 1732433748068) 2024-11-24T07:35:48,153 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestAsyncWALReplayValueCompression#testCompactedBulkLoadedFiles Thread=442 (was 428) Potentially hanging thread: AsyncFSWAL-26-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Client (1319272049) connection to localhost/127.0.0.1:33505 from jenkinstestCompactedBulkLoadedFiles java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) Potentially hanging thread: AsyncFSWAL-26-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_2023082271_22 at /127.0.0.1:51274 [Waiting for operation #5] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_2023082271_22 at /127.0.0.1:57494 [Waiting for operation #7] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-26-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: LeaseRenewer:jenkinstestCompactedBulkLoadedFiles@localhost:33505 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_2023082271_22 at /127.0.0.1:39394 [Waiting for operation #7] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=1348 (was 1264) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=169 (was 184), ProcessCount=11 (was 11), AvailableMemoryMB=1927 (was 1924) - AvailableMemoryMB LEAK? - 2024-11-24T07:35:48,153 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1348 is superior to 1024 2024-11-24T07:35:48,164 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestAsyncWALReplayValueCompression#testReplayEditsWrittenViaHRegion Thread=442, OpenFileDescriptor=1348, MaxFileDescriptor=1048576, SystemLoadAverage=169, ProcessCount=11, AvailableMemoryMB=1925 2024-11-24T07:35:48,164 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1348 is superior to 1024 2024-11-24T07:35:48,178 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-24T07:35:48,179 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-24T07:35:48,180 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-24T07:35:48,182 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-81185400, suffix=, logDir=hdfs://localhost:33505/hbase/WALs/hregion-81185400, archiveDir=hdfs://localhost:33505/hbase/oldWALs, maxLogs=32 2024-11-24T07:35:48,194 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-81185400/hregion-81185400.1732433748183, exclude list is [], retry=0 2024-11-24T07:35:48,197 DEBUG [AsyncFSWAL-28-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:42685,DS-467ee9a1-f3ca-4279-9b61-69a2a6a0308f,DISK] 2024-11-24T07:35:48,197 DEBUG [AsyncFSWAL-28-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41161,DS-b0260eac-1380-4e84-bf22-a60109fdf5f6,DISK] 2024-11-24T07:35:48,197 DEBUG [AsyncFSWAL-28-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41971,DS-a67a12dc-5f72-4d11-ac5a-3e225e33c62a,DISK] 2024-11-24T07:35:48,199 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-81185400/hregion-81185400.1732433748183 2024-11-24T07:35:48,199 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:35537:35537),(127.0.0.1/127.0.0.1:40647:40647),(127.0.0.1/127.0.0.1:37111:37111)] 2024-11-24T07:35:48,199 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => 81428a0981749218a046eb79f7b3a811, NAME => 'testReplayEditsWrittenViaHRegion,,1732433748178.81428a0981749218a046eb79f7b3a811.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testReplayEditsWrittenViaHRegion', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:33505/hbase 2024-11-24T07:35:48,207 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41971 is added to blk_1073741931_1110 (size=67) 2024-11-24T07:35:48,207 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41161 is added to blk_1073741931_1110 (size=67) 2024-11-24T07:35:48,207 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42685 is added to blk_1073741931_1110 (size=67) 2024-11-24T07:35:48,208 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenViaHRegion,,1732433748178.81428a0981749218a046eb79f7b3a811.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T07:35:48,209 INFO [StoreOpener-81428a0981749218a046eb79f7b3a811-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 81428a0981749218a046eb79f7b3a811 2024-11-24T07:35:48,210 INFO [StoreOpener-81428a0981749218a046eb79f7b3a811-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 81428a0981749218a046eb79f7b3a811 columnFamilyName a 2024-11-24T07:35:48,210 DEBUG [StoreOpener-81428a0981749218a046eb79f7b3a811-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T07:35:48,211 INFO [StoreOpener-81428a0981749218a046eb79f7b3a811-1 {}] regionserver.HStore(327): Store=81428a0981749218a046eb79f7b3a811/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T07:35:48,211 INFO [StoreOpener-81428a0981749218a046eb79f7b3a811-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 81428a0981749218a046eb79f7b3a811 2024-11-24T07:35:48,212 INFO [StoreOpener-81428a0981749218a046eb79f7b3a811-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 81428a0981749218a046eb79f7b3a811 columnFamilyName b 2024-11-24T07:35:48,212 DEBUG [StoreOpener-81428a0981749218a046eb79f7b3a811-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T07:35:48,212 INFO [StoreOpener-81428a0981749218a046eb79f7b3a811-1 {}] regionserver.HStore(327): Store=81428a0981749218a046eb79f7b3a811/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T07:35:48,212 INFO [StoreOpener-81428a0981749218a046eb79f7b3a811-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 81428a0981749218a046eb79f7b3a811 2024-11-24T07:35:48,213 INFO [StoreOpener-81428a0981749218a046eb79f7b3a811-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 81428a0981749218a046eb79f7b3a811 columnFamilyName c 2024-11-24T07:35:48,213 DEBUG [StoreOpener-81428a0981749218a046eb79f7b3a811-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T07:35:48,214 INFO [StoreOpener-81428a0981749218a046eb79f7b3a811-1 {}] regionserver.HStore(327): Store=81428a0981749218a046eb79f7b3a811/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T07:35:48,214 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 81428a0981749218a046eb79f7b3a811 2024-11-24T07:35:48,214 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33505/hbase/data/default/testReplayEditsWrittenViaHRegion/81428a0981749218a046eb79f7b3a811 2024-11-24T07:35:48,215 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33505/hbase/data/default/testReplayEditsWrittenViaHRegion/81428a0981749218a046eb79f7b3a811 2024-11-24T07:35:48,216 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 81428a0981749218a046eb79f7b3a811 2024-11-24T07:35:48,216 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 81428a0981749218a046eb79f7b3a811 2024-11-24T07:35:48,216 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsWrittenViaHRegion descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-24T07:35:48,217 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 81428a0981749218a046eb79f7b3a811 2024-11-24T07:35:48,219 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:33505/hbase/data/default/testReplayEditsWrittenViaHRegion/81428a0981749218a046eb79f7b3a811/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-24T07:35:48,219 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 81428a0981749218a046eb79f7b3a811; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=73518147, jitterRate=0.09550575911998749}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-24T07:35:48,219 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 81428a0981749218a046eb79f7b3a811: Writing region info on filesystem at 1732433748208Initializing all the Stores at 1732433748208Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732433748209 (+1 ms)Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732433748209Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732433748209Cleaning up temporary data from old regions at 1732433748216 (+7 ms)Region opened successfully at 1732433748219 (+3 ms) 2024-11-24T07:35:48,219 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 81428a0981749218a046eb79f7b3a811, disabling compactions & flushes 2024-11-24T07:35:48,219 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsWrittenViaHRegion,,1732433748178.81428a0981749218a046eb79f7b3a811. 2024-11-24T07:35:48,219 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsWrittenViaHRegion,,1732433748178.81428a0981749218a046eb79f7b3a811. 2024-11-24T07:35:48,219 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsWrittenViaHRegion,,1732433748178.81428a0981749218a046eb79f7b3a811. after waiting 0 ms 2024-11-24T07:35:48,219 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsWrittenViaHRegion,,1732433748178.81428a0981749218a046eb79f7b3a811. 2024-11-24T07:35:48,220 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsWrittenViaHRegion,,1732433748178.81428a0981749218a046eb79f7b3a811. 2024-11-24T07:35:48,220 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 81428a0981749218a046eb79f7b3a811: Waiting for close lock at 1732433748219Disabling compacts and flushes for region at 1732433748219Disabling writes for close at 1732433748219Writing region close event to WAL at 1732433748220 (+1 ms)Closed at 1732433748220 2024-11-24T07:35:48,222 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41971 is added to blk_1073741930_1109 (size=95) 2024-11-24T07:35:48,222 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42685 is added to blk_1073741930_1109 (size=95) 2024-11-24T07:35:48,222 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41161 is added to blk_1073741930_1109 (size=95) 2024-11-24T07:35:48,224 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-11-24T07:35:48,224 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-81185400:(num 1732433748183) 2024-11-24T07:35:48,224 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-24T07:35:48,225 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:33505/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732433748177, archiveDir=hdfs://localhost:33505/hbase/oldWALs, maxLogs=32 2024-11-24T07:35:48,238 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732433748177/wal.1732433748226, exclude list is [], retry=0 2024-11-24T07:35:48,241 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41161,DS-b0260eac-1380-4e84-bf22-a60109fdf5f6,DISK] 2024-11-24T07:35:48,241 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:42685,DS-467ee9a1-f3ca-4279-9b61-69a2a6a0308f,DISK] 2024-11-24T07:35:48,241 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41971,DS-a67a12dc-5f72-4d11-ac5a-3e225e33c62a,DISK] 2024-11-24T07:35:48,243 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732433748177/wal.1732433748226 2024-11-24T07:35:48,243 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:40647:40647),(127.0.0.1/127.0.0.1:35537:35537),(127.0.0.1/127.0.0.1:37111:37111)] 2024-11-24T07:35:48,243 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 81428a0981749218a046eb79f7b3a811, NAME => 'testReplayEditsWrittenViaHRegion,,1732433748178.81428a0981749218a046eb79f7b3a811.', STARTKEY => '', ENDKEY => ''} 2024-11-24T07:35:48,243 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenViaHRegion,,1732433748178.81428a0981749218a046eb79f7b3a811.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T07:35:48,243 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 81428a0981749218a046eb79f7b3a811 2024-11-24T07:35:48,244 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 81428a0981749218a046eb79f7b3a811 2024-11-24T07:35:48,245 INFO [StoreOpener-81428a0981749218a046eb79f7b3a811-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 81428a0981749218a046eb79f7b3a811 2024-11-24T07:35:48,246 INFO [StoreOpener-81428a0981749218a046eb79f7b3a811-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 81428a0981749218a046eb79f7b3a811 columnFamilyName a 2024-11-24T07:35:48,246 DEBUG [StoreOpener-81428a0981749218a046eb79f7b3a811-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T07:35:48,246 INFO [StoreOpener-81428a0981749218a046eb79f7b3a811-1 {}] regionserver.HStore(327): Store=81428a0981749218a046eb79f7b3a811/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T07:35:48,247 INFO [StoreOpener-81428a0981749218a046eb79f7b3a811-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 81428a0981749218a046eb79f7b3a811 2024-11-24T07:35:48,247 INFO [StoreOpener-81428a0981749218a046eb79f7b3a811-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 81428a0981749218a046eb79f7b3a811 columnFamilyName b 2024-11-24T07:35:48,247 DEBUG [StoreOpener-81428a0981749218a046eb79f7b3a811-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T07:35:48,248 INFO [StoreOpener-81428a0981749218a046eb79f7b3a811-1 {}] regionserver.HStore(327): Store=81428a0981749218a046eb79f7b3a811/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T07:35:48,248 INFO [StoreOpener-81428a0981749218a046eb79f7b3a811-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 81428a0981749218a046eb79f7b3a811 2024-11-24T07:35:48,249 INFO [StoreOpener-81428a0981749218a046eb79f7b3a811-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 81428a0981749218a046eb79f7b3a811 columnFamilyName c 2024-11-24T07:35:48,249 DEBUG [StoreOpener-81428a0981749218a046eb79f7b3a811-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T07:35:48,249 INFO [StoreOpener-81428a0981749218a046eb79f7b3a811-1 {}] regionserver.HStore(327): Store=81428a0981749218a046eb79f7b3a811/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T07:35:48,249 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 81428a0981749218a046eb79f7b3a811 2024-11-24T07:35:48,250 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33505/hbase/data/default/testReplayEditsWrittenViaHRegion/81428a0981749218a046eb79f7b3a811 2024-11-24T07:35:48,251 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33505/hbase/data/default/testReplayEditsWrittenViaHRegion/81428a0981749218a046eb79f7b3a811 2024-11-24T07:35:48,252 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 81428a0981749218a046eb79f7b3a811 2024-11-24T07:35:48,252 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 81428a0981749218a046eb79f7b3a811 2024-11-24T07:35:48,252 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsWrittenViaHRegion descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-24T07:35:48,254 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 81428a0981749218a046eb79f7b3a811 2024-11-24T07:35:48,254 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 81428a0981749218a046eb79f7b3a811; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=71569440, jitterRate=0.0664677619934082}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-24T07:35:48,255 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 81428a0981749218a046eb79f7b3a811: Writing region info on filesystem at 1732433748244Initializing all the Stores at 1732433748245 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732433748245Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732433748245Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732433748245Cleaning up temporary data from old regions at 1732433748252 (+7 ms)Region opened successfully at 1732433748255 (+3 ms) 2024-11-24T07:35:48,262 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 81428a0981749218a046eb79f7b3a811 3/3 column families, dataSize=870 B heapSize=2.31 KB 2024-11-24T07:35:48,277 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33505/hbase/data/default/testReplayEditsWrittenViaHRegion/81428a0981749218a046eb79f7b3a811/.tmp/a/8d11b93d59454ce4b591009303b476f4 is 91, key is testReplayEditsWrittenViaHRegion/a:x0/1732433748255/Put/seqid=0 2024-11-24T07:35:48,283 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41161 is added to blk_1073741933_1112 (size=5958) 2024-11-24T07:35:48,283 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41971 is added to blk_1073741933_1112 (size=5958) 2024-11-24T07:35:48,284 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42685 is added to blk_1073741933_1112 (size=5958) 2024-11-24T07:35:48,284 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:33505/hbase/data/default/testReplayEditsWrittenViaHRegion/81428a0981749218a046eb79f7b3a811/.tmp/a/8d11b93d59454ce4b591009303b476f4 2024-11-24T07:35:48,289 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33505/hbase/data/default/testReplayEditsWrittenViaHRegion/81428a0981749218a046eb79f7b3a811/.tmp/a/8d11b93d59454ce4b591009303b476f4 as hdfs://localhost:33505/hbase/data/default/testReplayEditsWrittenViaHRegion/81428a0981749218a046eb79f7b3a811/a/8d11b93d59454ce4b591009303b476f4 2024-11-24T07:35:48,294 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33505/hbase/data/default/testReplayEditsWrittenViaHRegion/81428a0981749218a046eb79f7b3a811/a/8d11b93d59454ce4b591009303b476f4, entries=10, sequenceid=13, filesize=5.8 K 2024-11-24T07:35:48,295 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~870 B/870, heapSize ~1.80 KB/1840, currentSize=0 B/0 for 81428a0981749218a046eb79f7b3a811 in 33ms, sequenceid=13, compaction requested=false 2024-11-24T07:35:48,296 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for 81428a0981749218a046eb79f7b3a811: 2024-11-24T07:35:48,310 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 81428a0981749218a046eb79f7b3a811, disabling compactions & flushes 2024-11-24T07:35:48,311 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsWrittenViaHRegion,,1732433748178.81428a0981749218a046eb79f7b3a811. 2024-11-24T07:35:48,311 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsWrittenViaHRegion,,1732433748178.81428a0981749218a046eb79f7b3a811. 2024-11-24T07:35:48,311 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsWrittenViaHRegion,,1732433748178.81428a0981749218a046eb79f7b3a811. after waiting 0 ms 2024-11-24T07:35:48,311 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsWrittenViaHRegion,,1732433748178.81428a0981749218a046eb79f7b3a811. 2024-11-24T07:35:48,311 ERROR [Time-limited test {}] regionserver.HRegion(1960): Memstore data size is 1740 in region testReplayEditsWrittenViaHRegion,,1732433748178.81428a0981749218a046eb79f7b3a811. 2024-11-24T07:35:48,312 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsWrittenViaHRegion,,1732433748178.81428a0981749218a046eb79f7b3a811. 2024-11-24T07:35:48,312 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 81428a0981749218a046eb79f7b3a811: Waiting for close lock at 1732433748310Disabling compacts and flushes for region at 1732433748310Disabling writes for close at 1732433748311 (+1 ms)Writing region close event to WAL at 1732433748311Closed at 1732433748311 2024-11-24T07:35:48,314 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42685 is added to blk_1073741932_1111 (size=3344) 2024-11-24T07:35:48,314 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41161 is added to blk_1073741932_1111 (size=3344) 2024-11-24T07:35:48,314 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41971 is added to blk_1073741932_1111 (size=3344) 2024-11-24T07:35:48,329 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:33505/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732433748177/wal.1732433748226, size=3.3 K (3344bytes) 2024-11-24T07:35:48,329 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:33505/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732433748177/wal.1732433748226 2024-11-24T07:35:48,330 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:33505/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732433748177/wal.1732433748226 after 1ms 2024-11-24T07:35:48,332 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:33505/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732433748177/wal.1732433748226: isRecoveredEdits=false, hasTagCompression=true, hasValueCompression=true, valueCompressionType=GZ 2024-11-24T07:35:48,332 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:33505/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732433748177/wal.1732433748226 took 3ms 2024-11-24T07:35:48,334 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:33505/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732433748177/wal.1732433748226 so closing down 2024-11-24T07:35:48,334 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-24T07:35:48,335 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000003-wal.1732433748226.temp 2024-11-24T07:35:48,336 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:33505/hbase/data/default/testReplayEditsWrittenViaHRegion/81428a0981749218a046eb79f7b3a811/recovered.edits/0000000000000000003-wal.1732433748226.temp 2024-11-24T07:35:48,337 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-11-24T07:35:48,342 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42685 is added to blk_1073741934_1113 (size=2944) 2024-11-24T07:35:48,342 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41161 is added to blk_1073741934_1113 (size=2944) 2024-11-24T07:35:48,342 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41971 is added to blk_1073741934_1113 (size=2944) 2024-11-24T07:35:48,345 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:33505/hbase/data/default/testReplayEditsWrittenViaHRegion/81428a0981749218a046eb79f7b3a811/recovered.edits/0000000000000000003-wal.1732433748226.temp (wrote 30 edits, skipped 0 edits in 0 ms) 2024-11-24T07:35:48,346 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:33505/hbase/data/default/testReplayEditsWrittenViaHRegion/81428a0981749218a046eb79f7b3a811/recovered.edits/0000000000000000003-wal.1732433748226.temp to hdfs://localhost:33505/hbase/data/default/testReplayEditsWrittenViaHRegion/81428a0981749218a046eb79f7b3a811/recovered.edits/0000000000000000035 2024-11-24T07:35:48,347 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 32 edits across 1 Regions in 14 ms; skipped=2; WAL=hdfs://localhost:33505/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732433748177/wal.1732433748226, size=3.3 K, length=3344, corrupted=false, cancelled=false 2024-11-24T07:35:48,347 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:33505/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732433748177/wal.1732433748226, journal: Splitting hdfs://localhost:33505/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732433748177/wal.1732433748226, size=3.3 K (3344bytes) at 1732433748329Finishing writing output for hdfs://localhost:33505/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732433748177/wal.1732433748226 so closing down at 1732433748334 (+5 ms)Creating recovered edits writer path=hdfs://localhost:33505/hbase/data/default/testReplayEditsWrittenViaHRegion/81428a0981749218a046eb79f7b3a811/recovered.edits/0000000000000000003-wal.1732433748226.temp at 1732433748336 (+2 ms)3 split writer threads finished at 1732433748337 (+1 ms)Closed recovered edits writer path=hdfs://localhost:33505/hbase/data/default/testReplayEditsWrittenViaHRegion/81428a0981749218a046eb79f7b3a811/recovered.edits/0000000000000000003-wal.1732433748226.temp (wrote 30 edits, skipped 0 edits in 0 ms) at 1732433748345 (+8 ms)Rename recovered edits hdfs://localhost:33505/hbase/data/default/testReplayEditsWrittenViaHRegion/81428a0981749218a046eb79f7b3a811/recovered.edits/0000000000000000003-wal.1732433748226.temp to hdfs://localhost:33505/hbase/data/default/testReplayEditsWrittenViaHRegion/81428a0981749218a046eb79f7b3a811/recovered.edits/0000000000000000035 at 1732433748346 (+1 ms)Processed 32 edits across 1 Regions in 14 ms; skipped=2; WAL=hdfs://localhost:33505/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732433748177/wal.1732433748226, size=3.3 K, length=3344, corrupted=false, cancelled=false at 1732433748347 (+1 ms) 2024-11-24T07:35:48,348 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:33505/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732433748177/wal.1732433748226 to hdfs://localhost:33505/hbase/oldWALs/wal.1732433748226 2024-11-24T07:35:48,349 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:33505/hbase/data/default/testReplayEditsWrittenViaHRegion/81428a0981749218a046eb79f7b3a811/recovered.edits/0000000000000000035 2024-11-24T07:35:48,349 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-24T07:35:48,351 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:33505/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732433748177, archiveDir=hdfs://localhost:33505/hbase/oldWALs, maxLogs=32 2024-11-24T07:35:48,362 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732433748177/wal.1732433748351, exclude list is [], retry=0 2024-11-24T07:35:48,365 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41971,DS-a67a12dc-5f72-4d11-ac5a-3e225e33c62a,DISK] 2024-11-24T07:35:48,365 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:42685,DS-467ee9a1-f3ca-4279-9b61-69a2a6a0308f,DISK] 2024-11-24T07:35:48,365 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41161,DS-b0260eac-1380-4e84-bf22-a60109fdf5f6,DISK] 2024-11-24T07:35:48,366 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732433748177/wal.1732433748351 2024-11-24T07:35:48,367 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:37111:37111),(127.0.0.1/127.0.0.1:35537:35537),(127.0.0.1/127.0.0.1:40647:40647)] 2024-11-24T07:35:48,367 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 81428a0981749218a046eb79f7b3a811, NAME => 'testReplayEditsWrittenViaHRegion,,1732433748178.81428a0981749218a046eb79f7b3a811.', STARTKEY => '', ENDKEY => ''} 2024-11-24T07:35:48,367 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenViaHRegion,,1732433748178.81428a0981749218a046eb79f7b3a811.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T07:35:48,367 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 81428a0981749218a046eb79f7b3a811 2024-11-24T07:35:48,367 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 81428a0981749218a046eb79f7b3a811 2024-11-24T07:35:48,368 INFO [StoreOpener-81428a0981749218a046eb79f7b3a811-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 81428a0981749218a046eb79f7b3a811 2024-11-24T07:35:48,369 INFO [StoreOpener-81428a0981749218a046eb79f7b3a811-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 81428a0981749218a046eb79f7b3a811 columnFamilyName a 2024-11-24T07:35:48,369 DEBUG [StoreOpener-81428a0981749218a046eb79f7b3a811-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T07:35:48,376 DEBUG [StoreOpener-81428a0981749218a046eb79f7b3a811-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:33505/hbase/data/default/testReplayEditsWrittenViaHRegion/81428a0981749218a046eb79f7b3a811/a/8d11b93d59454ce4b591009303b476f4 2024-11-24T07:35:48,376 INFO [StoreOpener-81428a0981749218a046eb79f7b3a811-1 {}] regionserver.HStore(327): Store=81428a0981749218a046eb79f7b3a811/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T07:35:48,376 INFO [StoreOpener-81428a0981749218a046eb79f7b3a811-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 81428a0981749218a046eb79f7b3a811 2024-11-24T07:35:48,377 INFO [StoreOpener-81428a0981749218a046eb79f7b3a811-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 81428a0981749218a046eb79f7b3a811 columnFamilyName b 2024-11-24T07:35:48,377 DEBUG [StoreOpener-81428a0981749218a046eb79f7b3a811-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T07:35:48,378 INFO [StoreOpener-81428a0981749218a046eb79f7b3a811-1 {}] regionserver.HStore(327): Store=81428a0981749218a046eb79f7b3a811/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T07:35:48,378 INFO [StoreOpener-81428a0981749218a046eb79f7b3a811-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 81428a0981749218a046eb79f7b3a811 2024-11-24T07:35:48,379 INFO [StoreOpener-81428a0981749218a046eb79f7b3a811-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 81428a0981749218a046eb79f7b3a811 columnFamilyName c 2024-11-24T07:35:48,379 DEBUG [StoreOpener-81428a0981749218a046eb79f7b3a811-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T07:35:48,379 INFO [StoreOpener-81428a0981749218a046eb79f7b3a811-1 {}] regionserver.HStore(327): Store=81428a0981749218a046eb79f7b3a811/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T07:35:48,379 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 81428a0981749218a046eb79f7b3a811 2024-11-24T07:35:48,380 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33505/hbase/data/default/testReplayEditsWrittenViaHRegion/81428a0981749218a046eb79f7b3a811 2024-11-24T07:35:48,381 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:33505/hbase/data/default/testReplayEditsWrittenViaHRegion/81428a0981749218a046eb79f7b3a811 2024-11-24T07:35:48,382 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:33505/hbase/data/default/testReplayEditsWrittenViaHRegion/81428a0981749218a046eb79f7b3a811/recovered.edits/0000000000000000035 2024-11-24T07:35:48,383 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:33505/hbase/data/default/testReplayEditsWrittenViaHRegion/81428a0981749218a046eb79f7b3a811/recovered.edits/0000000000000000035: isRecoveredEdits=true, hasTagCompression=true, hasValueCompression=true, valueCompressionType=GZ 2024-11-24T07:35:48,384 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 20, skipped 10, firstSequenceIdInLog=3, maxSequenceIdInLog=35, path=hdfs://localhost:33505/hbase/data/default/testReplayEditsWrittenViaHRegion/81428a0981749218a046eb79f7b3a811/recovered.edits/0000000000000000035 2024-11-24T07:35:48,385 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 81428a0981749218a046eb79f7b3a811 3/3 column families, dataSize=1.70 KB heapSize=3.88 KB 2024-11-24T07:35:48,397 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-24T07:35:48,398 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33505/hbase/data/default/testReplayEditsWrittenViaHRegion/81428a0981749218a046eb79f7b3a811/.tmp/b/ab27f6c18f3a44b89796247cad84559d is 91, key is testReplayEditsWrittenViaHRegion/b:x0/1732433748296/Put/seqid=0 2024-11-24T07:35:48,407 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42685 is added to blk_1073741936_1115 (size=5958) 2024-11-24T07:35:48,407 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41971 is added to blk_1073741936_1115 (size=5958) 2024-11-24T07:35:48,408 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41161 is added to blk_1073741936_1115 (size=5958) 2024-11-24T07:35:48,409 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=35 (bloomFilter=true), to=hdfs://localhost:33505/hbase/data/default/testReplayEditsWrittenViaHRegion/81428a0981749218a046eb79f7b3a811/.tmp/b/ab27f6c18f3a44b89796247cad84559d 2024-11-24T07:35:48,433 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33505/hbase/data/default/testReplayEditsWrittenViaHRegion/81428a0981749218a046eb79f7b3a811/.tmp/c/c436f7f6d0214718ade08af671e48688 is 91, key is testReplayEditsWrittenViaHRegion/c:x0/1732433748302/Put/seqid=0 2024-11-24T07:35:48,438 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41161 is added to blk_1073741937_1116 (size=5958) 2024-11-24T07:35:48,439 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42685 is added to blk_1073741937_1116 (size=5958) 2024-11-24T07:35:48,439 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41971 is added to blk_1073741937_1116 (size=5958) 2024-11-24T07:35:48,439 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=35 (bloomFilter=true), to=hdfs://localhost:33505/hbase/data/default/testReplayEditsWrittenViaHRegion/81428a0981749218a046eb79f7b3a811/.tmp/c/c436f7f6d0214718ade08af671e48688 2024-11-24T07:35:48,445 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33505/hbase/data/default/testReplayEditsWrittenViaHRegion/81428a0981749218a046eb79f7b3a811/.tmp/b/ab27f6c18f3a44b89796247cad84559d as hdfs://localhost:33505/hbase/data/default/testReplayEditsWrittenViaHRegion/81428a0981749218a046eb79f7b3a811/b/ab27f6c18f3a44b89796247cad84559d 2024-11-24T07:35:48,450 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33505/hbase/data/default/testReplayEditsWrittenViaHRegion/81428a0981749218a046eb79f7b3a811/b/ab27f6c18f3a44b89796247cad84559d, entries=10, sequenceid=35, filesize=5.8 K 2024-11-24T07:35:48,451 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33505/hbase/data/default/testReplayEditsWrittenViaHRegion/81428a0981749218a046eb79f7b3a811/.tmp/c/c436f7f6d0214718ade08af671e48688 as hdfs://localhost:33505/hbase/data/default/testReplayEditsWrittenViaHRegion/81428a0981749218a046eb79f7b3a811/c/c436f7f6d0214718ade08af671e48688 2024-11-24T07:35:48,455 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33505/hbase/data/default/testReplayEditsWrittenViaHRegion/81428a0981749218a046eb79f7b3a811/c/c436f7f6d0214718ade08af671e48688, entries=10, sequenceid=35, filesize=5.8 K 2024-11-24T07:35:48,455 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~1.70 KB/1740, heapSize ~3.59 KB/3680, currentSize=0 B/0 for 81428a0981749218a046eb79f7b3a811 in 70ms, sequenceid=35, compaction requested=false; wal=null 2024-11-24T07:35:48,456 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:33505/hbase/data/default/testReplayEditsWrittenViaHRegion/81428a0981749218a046eb79f7b3a811/recovered.edits/0000000000000000035 2024-11-24T07:35:48,457 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 81428a0981749218a046eb79f7b3a811 2024-11-24T07:35:48,457 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 81428a0981749218a046eb79f7b3a811 2024-11-24T07:35:48,457 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsWrittenViaHRegion descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-24T07:35:48,458 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 81428a0981749218a046eb79f7b3a811 2024-11-24T07:35:48,461 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:33505/hbase/data/default/testReplayEditsWrittenViaHRegion/81428a0981749218a046eb79f7b3a811/recovered.edits/35.seqid, newMaxSeqId=35, maxSeqId=1 2024-11-24T07:35:48,461 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 81428a0981749218a046eb79f7b3a811; next sequenceid=36; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=71606278, jitterRate=0.06701669096946716}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-24T07:35:48,462 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 81428a0981749218a046eb79f7b3a811: Writing region info on filesystem at 1732433748367Initializing all the Stores at 1732433748368 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732433748368Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732433748368Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732433748368Obtaining lock to block concurrent updates at 1732433748385 (+17 ms)Preparing flush snapshotting stores in 81428a0981749218a046eb79f7b3a811 at 1732433748385Finished memstore snapshotting testReplayEditsWrittenViaHRegion,,1732433748178.81428a0981749218a046eb79f7b3a811., syncing WAL and waiting on mvcc, flushsize=dataSize=1740, getHeapSize=3920, getOffHeapSize=0, getCellsCount=20 at 1732433748385Flushing stores of testReplayEditsWrittenViaHRegion,,1732433748178.81428a0981749218a046eb79f7b3a811. at 1732433748385Flushing 81428a0981749218a046eb79f7b3a811/b: creating writer at 1732433748385Flushing 81428a0981749218a046eb79f7b3a811/b: appending metadata at 1732433748397 (+12 ms)Flushing 81428a0981749218a046eb79f7b3a811/b: closing flushed file at 1732433748398 (+1 ms)Flushing 81428a0981749218a046eb79f7b3a811/c: creating writer at 1732433748414 (+16 ms)Flushing 81428a0981749218a046eb79f7b3a811/c: appending metadata at 1732433748432 (+18 ms)Flushing 81428a0981749218a046eb79f7b3a811/c: closing flushed file at 1732433748432Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@624932d3: reopening flushed file at 1732433748444 (+12 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@1e827696: reopening flushed file at 1732433748450 (+6 ms)Finished flush of dataSize ~1.70 KB/1740, heapSize ~3.59 KB/3680, currentSize=0 B/0 for 81428a0981749218a046eb79f7b3a811 in 70ms, sequenceid=35, compaction requested=false; wal=null at 1732433748455 (+5 ms)Cleaning up temporary data from old regions at 1732433748457 (+2 ms)Region opened successfully at 1732433748461 (+4 ms) 2024-11-24T07:35:48,523 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:33505/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732433748177/wal.1732433748351, size=0 (0bytes) 2024-11-24T07:35:48,523 WARN [Time-limited test {}] wal.WALSplitter(453): File hdfs://localhost:33505/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732433748177/wal.1732433748351 might be still open, length is 0 2024-11-24T07:35:48,523 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:33505/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732433748177/wal.1732433748351 2024-11-24T07:35:48,524 WARN [IPC Server handler 0 on default port 33505 {}] namenode.FSNamesystem(3854): DIR* NameSystem.internalReleaseLease: File /hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732433748177/wal.1732433748351 has not been closed. Lease recovery is in progress. RecoveryId = 1117 for block blk_1073741935_1114 2024-11-24T07:35:48,524 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=0 on file=hdfs://localhost:33505/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732433748177/wal.1732433748351 after 1ms 2024-11-24T07:35:50,855 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-166876350_22 at /127.0.0.1:39444 [Receiving block BP-1337235608-172.17.0.3-1732433708251:blk_1073741935_1114] {}] datanode.DataXceiver(331): 127.0.0.1:41971:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:39444 dst: /127.0.0.1:41971 java.io.InterruptedIOException: Interrupted while waiting for IO on channel java.nio.channels.SocketChannel[connected local=localhost/127.0.0.1:41971 remote=/127.0.0.1:39444]. Total timeout mills is 60000, 57631 millis timeout left. at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:350) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?] at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?] at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?] at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?] at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?] at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T07:35:50,857 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-166876350_22 at /127.0.0.1:57576 [Receiving block BP-1337235608-172.17.0.3-1732433708251:blk_1073741935_1114] {}] datanode.DataXceiver(331): 127.0.0.1:42685:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:57576 dst: /127.0.0.1:42685 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T07:35:50,857 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-166876350_22 at /127.0.0.1:51332 [Receiving block BP-1337235608-172.17.0.3-1732433708251:blk_1073741935_1114] {}] datanode.DataXceiver(331): 127.0.0.1:41161:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:51332 dst: /127.0.0.1:41161 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T07:35:50,862 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41971 is added to blk_1073741935_1117 (size=2936) 2024-11-24T07:35:50,862 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42685 is added to blk_1073741935_1117 (size=2936) 2024-11-24T07:35:50,863 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41161 is added to blk_1073741935_1117 (size=2936) 2024-11-24T07:35:52,525 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=1 on file=hdfs://localhost:33505/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732433748177/wal.1732433748351 after 4002ms 2024-11-24T07:35:52,529 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:33505/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732433748177/wal.1732433748351: isRecoveredEdits=false, hasTagCompression=true, hasValueCompression=true, valueCompressionType=GZ 2024-11-24T07:35:52,529 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:33505/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732433748177/wal.1732433748351 took 4006ms 2024-11-24T07:35:52,533 INFO [Time-limited test {}] wal.WALSplitter(493): EOF from hdfs://localhost:33505/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732433748177/wal.1732433748351; continuing. 2024-11-24T07:35:52,533 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:33505/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732433748177/wal.1732433748351 so closing down 2024-11-24T07:35:52,533 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-24T07:35:52,535 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000037-wal.1732433748351.temp 2024-11-24T07:35:52,537 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:33505/hbase/data/default/testReplayEditsWrittenViaHRegion/81428a0981749218a046eb79f7b3a811/recovered.edits/0000000000000000037-wal.1732433748351.temp 2024-11-24T07:35:52,537 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-11-24T07:35:52,542 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41971 is added to blk_1073741938_1118 (size=2944) 2024-11-24T07:35:52,542 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42685 is added to blk_1073741938_1118 (size=2944) 2024-11-24T07:35:52,543 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41161 is added to blk_1073741938_1118 (size=2944) 2024-11-24T07:35:52,543 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:33505/hbase/data/default/testReplayEditsWrittenViaHRegion/81428a0981749218a046eb79f7b3a811/recovered.edits/0000000000000000037-wal.1732433748351.temp (wrote 30 edits, skipped 0 edits in 0 ms) 2024-11-24T07:35:52,544 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:33505/hbase/data/default/testReplayEditsWrittenViaHRegion/81428a0981749218a046eb79f7b3a811/recovered.edits/0000000000000000037-wal.1732433748351.temp to hdfs://localhost:33505/hbase/data/default/testReplayEditsWrittenViaHRegion/81428a0981749218a046eb79f7b3a811/recovered.edits/0000000000000000066 2024-11-24T07:35:52,544 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 30 edits across 1 Regions in 14 ms; skipped=0; WAL=hdfs://localhost:33505/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732433748177/wal.1732433748351, size=0, length=0, corrupted=false, cancelled=false 2024-11-24T07:35:52,544 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:33505/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732433748177/wal.1732433748351, journal: Splitting hdfs://localhost:33505/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732433748177/wal.1732433748351, size=0 (0bytes) at 1732433748523Finishing writing output for hdfs://localhost:33505/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732433748177/wal.1732433748351 so closing down at 1732433752533 (+4010 ms)Creating recovered edits writer path=hdfs://localhost:33505/hbase/data/default/testReplayEditsWrittenViaHRegion/81428a0981749218a046eb79f7b3a811/recovered.edits/0000000000000000037-wal.1732433748351.temp at 1732433752537 (+4 ms)3 split writer threads finished at 1732433752537Closed recovered edits writer path=hdfs://localhost:33505/hbase/data/default/testReplayEditsWrittenViaHRegion/81428a0981749218a046eb79f7b3a811/recovered.edits/0000000000000000037-wal.1732433748351.temp (wrote 30 edits, skipped 0 edits in 0 ms) at 1732433752543 (+6 ms)Rename recovered edits hdfs://localhost:33505/hbase/data/default/testReplayEditsWrittenViaHRegion/81428a0981749218a046eb79f7b3a811/recovered.edits/0000000000000000037-wal.1732433748351.temp to hdfs://localhost:33505/hbase/data/default/testReplayEditsWrittenViaHRegion/81428a0981749218a046eb79f7b3a811/recovered.edits/0000000000000000066 at 1732433752544 (+1 ms)Processed 30 edits across 1 Regions in 14 ms; skipped=0; WAL=hdfs://localhost:33505/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732433748177/wal.1732433748351, size=0, length=0, corrupted=false, cancelled=false at 1732433752544 2024-11-24T07:35:52,546 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:33505/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732433748177/wal.1732433748351 to hdfs://localhost:33505/hbase/oldWALs/wal.1732433748351 2024-11-24T07:35:52,547 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:33505/hbase/data/default/testReplayEditsWrittenViaHRegion/81428a0981749218a046eb79f7b3a811/recovered.edits/0000000000000000066 2024-11-24T07:35:52,547 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-24T07:35:52,548 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:33505/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732433748177, archiveDir=hdfs://localhost:33505/hbase/oldWALs, maxLogs=32 2024-11-24T07:35:52,561 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732433748177/wal.1732433752548, exclude list is [], retry=0 2024-11-24T07:35:52,563 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:42685,DS-467ee9a1-f3ca-4279-9b61-69a2a6a0308f,DISK] 2024-11-24T07:35:52,563 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41161,DS-b0260eac-1380-4e84-bf22-a60109fdf5f6,DISK] 2024-11-24T07:35:52,564 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41971,DS-a67a12dc-5f72-4d11-ac5a-3e225e33c62a,DISK] 2024-11-24T07:35:52,565 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732433748177/wal.1732433752548 2024-11-24T07:35:52,565 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:35537:35537),(127.0.0.1/127.0.0.1:40647:40647),(127.0.0.1/127.0.0.1:37111:37111)] 2024-11-24T07:35:52,566 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenViaHRegion,,1732433748178.81428a0981749218a046eb79f7b3a811.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T07:35:52,567 INFO [StoreOpener-81428a0981749218a046eb79f7b3a811-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 81428a0981749218a046eb79f7b3a811 2024-11-24T07:35:52,567 INFO [StoreOpener-81428a0981749218a046eb79f7b3a811-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 81428a0981749218a046eb79f7b3a811 columnFamilyName a 2024-11-24T07:35:52,568 DEBUG [StoreOpener-81428a0981749218a046eb79f7b3a811-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T07:35:52,575 DEBUG [StoreOpener-81428a0981749218a046eb79f7b3a811-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:33505/hbase/data/default/testReplayEditsWrittenViaHRegion/81428a0981749218a046eb79f7b3a811/a/8d11b93d59454ce4b591009303b476f4 2024-11-24T07:35:52,575 INFO [StoreOpener-81428a0981749218a046eb79f7b3a811-1 {}] regionserver.HStore(327): Store=81428a0981749218a046eb79f7b3a811/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T07:35:52,576 INFO [StoreOpener-81428a0981749218a046eb79f7b3a811-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 81428a0981749218a046eb79f7b3a811 2024-11-24T07:35:52,576 INFO [StoreOpener-81428a0981749218a046eb79f7b3a811-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 81428a0981749218a046eb79f7b3a811 columnFamilyName b 2024-11-24T07:35:52,576 DEBUG [StoreOpener-81428a0981749218a046eb79f7b3a811-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T07:35:52,581 DEBUG [StoreOpener-81428a0981749218a046eb79f7b3a811-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:33505/hbase/data/default/testReplayEditsWrittenViaHRegion/81428a0981749218a046eb79f7b3a811/b/ab27f6c18f3a44b89796247cad84559d 2024-11-24T07:35:52,582 INFO [StoreOpener-81428a0981749218a046eb79f7b3a811-1 {}] regionserver.HStore(327): Store=81428a0981749218a046eb79f7b3a811/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T07:35:52,582 INFO [StoreOpener-81428a0981749218a046eb79f7b3a811-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 81428a0981749218a046eb79f7b3a811 2024-11-24T07:35:52,583 INFO [StoreOpener-81428a0981749218a046eb79f7b3a811-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 81428a0981749218a046eb79f7b3a811 columnFamilyName c 2024-11-24T07:35:52,583 DEBUG [StoreOpener-81428a0981749218a046eb79f7b3a811-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T07:35:52,587 DEBUG [StoreOpener-81428a0981749218a046eb79f7b3a811-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:33505/hbase/data/default/testReplayEditsWrittenViaHRegion/81428a0981749218a046eb79f7b3a811/c/c436f7f6d0214718ade08af671e48688 2024-11-24T07:35:52,587 INFO [StoreOpener-81428a0981749218a046eb79f7b3a811-1 {}] regionserver.HStore(327): Store=81428a0981749218a046eb79f7b3a811/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T07:35:52,588 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 81428a0981749218a046eb79f7b3a811 2024-11-24T07:35:52,588 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33505/hbase/data/default/testReplayEditsWrittenViaHRegion/81428a0981749218a046eb79f7b3a811 2024-11-24T07:35:52,590 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:33505/hbase/data/default/testReplayEditsWrittenViaHRegion/81428a0981749218a046eb79f7b3a811 2024-11-24T07:35:52,591 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:33505/hbase/data/default/testReplayEditsWrittenViaHRegion/81428a0981749218a046eb79f7b3a811/recovered.edits/0000000000000000066 2024-11-24T07:35:52,592 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:33505/hbase/data/default/testReplayEditsWrittenViaHRegion/81428a0981749218a046eb79f7b3a811/recovered.edits/0000000000000000066: isRecoveredEdits=true, hasTagCompression=true, hasValueCompression=true, valueCompressionType=GZ 2024-11-24T07:35:52,596 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 30, skipped 0, firstSequenceIdInLog=37, maxSequenceIdInLog=66, path=hdfs://localhost:33505/hbase/data/default/testReplayEditsWrittenViaHRegion/81428a0981749218a046eb79f7b3a811/recovered.edits/0000000000000000066 2024-11-24T07:35:52,596 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 81428a0981749218a046eb79f7b3a811 3/3 column families, dataSize=2.55 KB heapSize=5.44 KB 2024-11-24T07:35:52,611 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33505/hbase/data/default/testReplayEditsWrittenViaHRegion/81428a0981749218a046eb79f7b3a811/.tmp/a/817705f9fa06444a87649f88ca0106a5 is 91, key is testReplayEditsWrittenViaHRegion/a:y0/1732433748467/Put/seqid=0 2024-11-24T07:35:52,616 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41161 is added to blk_1073741940_1120 (size=5958) 2024-11-24T07:35:52,617 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41971 is added to blk_1073741940_1120 (size=5958) 2024-11-24T07:35:52,617 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42685 is added to blk_1073741940_1120 (size=5958) 2024-11-24T07:35:52,617 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=66 (bloomFilter=true), to=hdfs://localhost:33505/hbase/data/default/testReplayEditsWrittenViaHRegion/81428a0981749218a046eb79f7b3a811/.tmp/a/817705f9fa06444a87649f88ca0106a5 2024-11-24T07:35:52,634 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33505/hbase/data/default/testReplayEditsWrittenViaHRegion/81428a0981749218a046eb79f7b3a811/.tmp/b/a522781e74b84a739987bb378d0efd2d is 91, key is testReplayEditsWrittenViaHRegion/b:y0/1732433748475/Put/seqid=0 2024-11-24T07:35:52,640 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41161 is added to blk_1073741941_1121 (size=5958) 2024-11-24T07:35:52,640 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41971 is added to blk_1073741941_1121 (size=5958) 2024-11-24T07:35:52,641 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42685 is added to blk_1073741941_1121 (size=5958) 2024-11-24T07:35:52,641 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=66 (bloomFilter=true), to=hdfs://localhost:33505/hbase/data/default/testReplayEditsWrittenViaHRegion/81428a0981749218a046eb79f7b3a811/.tmp/b/a522781e74b84a739987bb378d0efd2d 2024-11-24T07:35:52,659 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33505/hbase/data/default/testReplayEditsWrittenViaHRegion/81428a0981749218a046eb79f7b3a811/.tmp/c/b3e468e3dbf54f84b84b8a1c5cf1f31d is 91, key is testReplayEditsWrittenViaHRegion/c:y0/1732433748481/Put/seqid=0 2024-11-24T07:35:52,665 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41161 is added to blk_1073741942_1122 (size=5958) 2024-11-24T07:35:52,665 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42685 is added to blk_1073741942_1122 (size=5958) 2024-11-24T07:35:52,665 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41971 is added to blk_1073741942_1122 (size=5958) 2024-11-24T07:35:52,666 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=66 (bloomFilter=true), to=hdfs://localhost:33505/hbase/data/default/testReplayEditsWrittenViaHRegion/81428a0981749218a046eb79f7b3a811/.tmp/c/b3e468e3dbf54f84b84b8a1c5cf1f31d 2024-11-24T07:35:52,671 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33505/hbase/data/default/testReplayEditsWrittenViaHRegion/81428a0981749218a046eb79f7b3a811/.tmp/a/817705f9fa06444a87649f88ca0106a5 as hdfs://localhost:33505/hbase/data/default/testReplayEditsWrittenViaHRegion/81428a0981749218a046eb79f7b3a811/a/817705f9fa06444a87649f88ca0106a5 2024-11-24T07:35:52,677 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33505/hbase/data/default/testReplayEditsWrittenViaHRegion/81428a0981749218a046eb79f7b3a811/a/817705f9fa06444a87649f88ca0106a5, entries=10, sequenceid=66, filesize=5.8 K 2024-11-24T07:35:52,678 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33505/hbase/data/default/testReplayEditsWrittenViaHRegion/81428a0981749218a046eb79f7b3a811/.tmp/b/a522781e74b84a739987bb378d0efd2d as hdfs://localhost:33505/hbase/data/default/testReplayEditsWrittenViaHRegion/81428a0981749218a046eb79f7b3a811/b/a522781e74b84a739987bb378d0efd2d 2024-11-24T07:35:52,682 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33505/hbase/data/default/testReplayEditsWrittenViaHRegion/81428a0981749218a046eb79f7b3a811/b/a522781e74b84a739987bb378d0efd2d, entries=10, sequenceid=66, filesize=5.8 K 2024-11-24T07:35:52,683 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33505/hbase/data/default/testReplayEditsWrittenViaHRegion/81428a0981749218a046eb79f7b3a811/.tmp/c/b3e468e3dbf54f84b84b8a1c5cf1f31d as hdfs://localhost:33505/hbase/data/default/testReplayEditsWrittenViaHRegion/81428a0981749218a046eb79f7b3a811/c/b3e468e3dbf54f84b84b8a1c5cf1f31d 2024-11-24T07:35:52,689 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33505/hbase/data/default/testReplayEditsWrittenViaHRegion/81428a0981749218a046eb79f7b3a811/c/b3e468e3dbf54f84b84b8a1c5cf1f31d, entries=10, sequenceid=66, filesize=5.8 K 2024-11-24T07:35:52,689 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~2.55 KB/2610, heapSize ~5.39 KB/5520, currentSize=0 B/0 for 81428a0981749218a046eb79f7b3a811 in 93ms, sequenceid=66, compaction requested=false; wal=null 2024-11-24T07:35:52,690 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:33505/hbase/data/default/testReplayEditsWrittenViaHRegion/81428a0981749218a046eb79f7b3a811/recovered.edits/0000000000000000066 2024-11-24T07:35:52,691 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 81428a0981749218a046eb79f7b3a811 2024-11-24T07:35:52,691 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 81428a0981749218a046eb79f7b3a811 2024-11-24T07:35:52,691 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsWrittenViaHRegion descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-24T07:35:52,693 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 81428a0981749218a046eb79f7b3a811 2024-11-24T07:35:52,695 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:33505/hbase/data/default/testReplayEditsWrittenViaHRegion/81428a0981749218a046eb79f7b3a811/recovered.edits/66.seqid, newMaxSeqId=66, maxSeqId=35 2024-11-24T07:35:52,696 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 81428a0981749218a046eb79f7b3a811; next sequenceid=67; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=60415036, jitterRate=-0.09974581003189087}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-24T07:35:52,696 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 81428a0981749218a046eb79f7b3a811: Writing region info on filesystem at 1732433752566Initializing all the Stores at 1732433752566Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732433752566Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732433752567 (+1 ms)Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732433752567Obtaining lock to block concurrent updates at 1732433752596 (+29 ms)Preparing flush snapshotting stores in 81428a0981749218a046eb79f7b3a811 at 1732433752596Finished memstore snapshotting testReplayEditsWrittenViaHRegion,,1732433748178.81428a0981749218a046eb79f7b3a811., syncing WAL and waiting on mvcc, flushsize=dataSize=2610, getHeapSize=5520, getOffHeapSize=0, getCellsCount=30 at 1732433752596Flushing stores of testReplayEditsWrittenViaHRegion,,1732433748178.81428a0981749218a046eb79f7b3a811. at 1732433752596Flushing 81428a0981749218a046eb79f7b3a811/a: creating writer at 1732433752596Flushing 81428a0981749218a046eb79f7b3a811/a: appending metadata at 1732433752610 (+14 ms)Flushing 81428a0981749218a046eb79f7b3a811/a: closing flushed file at 1732433752610Flushing 81428a0981749218a046eb79f7b3a811/b: creating writer at 1732433752621 (+11 ms)Flushing 81428a0981749218a046eb79f7b3a811/b: appending metadata at 1732433752634 (+13 ms)Flushing 81428a0981749218a046eb79f7b3a811/b: closing flushed file at 1732433752634Flushing 81428a0981749218a046eb79f7b3a811/c: creating writer at 1732433752645 (+11 ms)Flushing 81428a0981749218a046eb79f7b3a811/c: appending metadata at 1732433752658 (+13 ms)Flushing 81428a0981749218a046eb79f7b3a811/c: closing flushed file at 1732433752658Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@9c191c7: reopening flushed file at 1732433752671 (+13 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@5adca800: reopening flushed file at 1732433752677 (+6 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@8bebe06: reopening flushed file at 1732433752683 (+6 ms)Finished flush of dataSize ~2.55 KB/2610, heapSize ~5.39 KB/5520, currentSize=0 B/0 for 81428a0981749218a046eb79f7b3a811 in 93ms, sequenceid=66, compaction requested=false; wal=null at 1732433752689 (+6 ms)Cleaning up temporary data from old regions at 1732433752691 (+2 ms)Region opened successfully at 1732433752696 (+5 ms) 2024-11-24T07:35:52,711 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 81428a0981749218a046eb79f7b3a811, disabling compactions & flushes 2024-11-24T07:35:52,711 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsWrittenViaHRegion,,1732433748178.81428a0981749218a046eb79f7b3a811. 2024-11-24T07:35:52,711 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsWrittenViaHRegion,,1732433748178.81428a0981749218a046eb79f7b3a811. 2024-11-24T07:35:52,711 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsWrittenViaHRegion,,1732433748178.81428a0981749218a046eb79f7b3a811. after waiting 0 ms 2024-11-24T07:35:52,711 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsWrittenViaHRegion,,1732433748178.81428a0981749218a046eb79f7b3a811. 2024-11-24T07:35:52,712 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsWrittenViaHRegion,,1732433748178.81428a0981749218a046eb79f7b3a811. 2024-11-24T07:35:52,712 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 81428a0981749218a046eb79f7b3a811: Waiting for close lock at 1732433752710Disabling compacts and flushes for region at 1732433752710Disabling writes for close at 1732433752711 (+1 ms)Writing region close event to WAL at 1732433752712 (+1 ms)Closed at 1732433752712 2024-11-24T07:35:52,715 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41971 is added to blk_1073741939_1119 (size=95) 2024-11-24T07:35:52,715 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41161 is added to blk_1073741939_1119 (size=95) 2024-11-24T07:35:52,715 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42685 is added to blk_1073741939_1119 (size=95) 2024-11-24T07:35:52,717 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-11-24T07:35:52,717 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL wal:(num 1732433752548) 2024-11-24T07:35:52,733 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestAsyncWALReplayValueCompression#testReplayEditsWrittenViaHRegion Thread=448 (was 442) Potentially hanging thread: AsyncFSWAL-28-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: LeaseRenewer:jenkinstestReplayEditsWrittenViaHRegion@localhost:33505 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1453160098_22 at /127.0.0.1:57794 [Waiting for operation #10] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-28-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Client (1319272049) connection to localhost/127.0.0.1:33505 from jenkinstestReplayEditsWrittenViaHRegion java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) Potentially hanging thread: AsyncFSWAL-28-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1453160098_22 at /127.0.0.1:60726 [Waiting for operation #15] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1453160098_22 at /127.0.0.1:33318 [Waiting for operation #18] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=1414 (was 1348) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=155 (was 169), ProcessCount=11 (was 11), AvailableMemoryMB=1911 (was 1925) 2024-11-24T07:35:52,733 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1414 is superior to 1024 2024-11-24T07:35:52,733 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster 2024-11-24T07:35:52,733 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-24T07:35:52,734 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79) at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611) at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-24T07:35:52,734 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T07:35:52,734 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T07:35:52,734 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-24T07:35:52,735 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster 2024-11-24T07:35:52,735 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=482707301, stopped=false 2024-11-24T07:35:52,735 INFO [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=1ab588ae992e,37719,1732433712713 2024-11-24T07:35:52,812 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40419-0x1016bc48eea0002, quorum=127.0.0.1:49244, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-24T07:35:52,812 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37719-0x1016bc48eea0000, quorum=127.0.0.1:49244, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-24T07:35:52,812 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46433-0x1016bc48eea0001, quorum=127.0.0.1:49244, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-24T07:35:52,812 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40419-0x1016bc48eea0002, quorum=127.0.0.1:49244, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-24T07:35:52,812 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37719-0x1016bc48eea0000, quorum=127.0.0.1:49244, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-24T07:35:52,812 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46433-0x1016bc48eea0001, quorum=127.0.0.1:49244, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-24T07:35:52,812 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-24T07:35:52,813 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-24T07:35:52,813 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277) at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265) at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416) at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-24T07:35:52,813 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T07:35:52,814 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:46433-0x1016bc48eea0001, quorum=127.0.0.1:49244, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-24T07:35:52,814 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:40419-0x1016bc48eea0002, quorum=127.0.0.1:49244, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-24T07:35:52,814 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '1ab588ae992e,46433,1732433713473' ***** 2024-11-24T07:35:52,815 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:37719-0x1016bc48eea0000, quorum=127.0.0.1:49244, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-24T07:35:52,815 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-11-24T07:35:52,815 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '1ab588ae992e,40419,1732433713579' ***** 2024-11-24T07:35:52,815 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-11-24T07:35:52,816 INFO [RS:1;1ab588ae992e:40419 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-24T07:35:52,816 INFO [RS:0;1ab588ae992e:46433 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-24T07:35:52,816 INFO [RS:1;1ab588ae992e:40419 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-24T07:35:52,816 INFO [RS:0;1ab588ae992e:46433 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-24T07:35:52,816 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-24T07:35:52,816 INFO [RS:0;1ab588ae992e:46433 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-24T07:35:52,816 INFO [RS:1;1ab588ae992e:40419 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-24T07:35:52,816 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-24T07:35:52,817 INFO [RS:0;1ab588ae992e:46433 {}] regionserver.HRegionServer(959): stopping server 1ab588ae992e,46433,1732433713473 2024-11-24T07:35:52,817 INFO [RS:1;1ab588ae992e:40419 {}] regionserver.HRegionServer(3091): Received CLOSE for 567d9cab2ff71509e601db9c0fa44478 2024-11-24T07:35:52,817 INFO [RS:0;1ab588ae992e:46433 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-24T07:35:52,817 INFO [RS:0;1ab588ae992e:46433 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;1ab588ae992e:46433. 2024-11-24T07:35:52,817 INFO [RS:1;1ab588ae992e:40419 {}] regionserver.HRegionServer(959): stopping server 1ab588ae992e,40419,1732433713579 2024-11-24T07:35:52,817 DEBUG [RS:0;1ab588ae992e:46433 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-24T07:35:52,817 INFO [RS:1;1ab588ae992e:40419 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-24T07:35:52,817 DEBUG [RS:0;1ab588ae992e:46433 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T07:35:52,817 INFO [RS:1;1ab588ae992e:40419 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:1;1ab588ae992e:40419. 2024-11-24T07:35:52,817 INFO [RS:0;1ab588ae992e:46433 {}] regionserver.HRegionServer(976): stopping server 1ab588ae992e,46433,1732433713473; all regions closed. 2024-11-24T07:35:52,817 DEBUG [RS_CLOSE_REGION-regionserver/1ab588ae992e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 567d9cab2ff71509e601db9c0fa44478, disabling compactions & flushes 2024-11-24T07:35:52,817 DEBUG [RS:1;1ab588ae992e:40419 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-24T07:35:52,818 DEBUG [RS:1;1ab588ae992e:40419 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T07:35:52,818 INFO [RS_CLOSE_REGION-regionserver/1ab588ae992e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testReplayEditsAfterRegionMovedWithMultiCF,,1732433728060.567d9cab2ff71509e601db9c0fa44478. 2024-11-24T07:35:52,818 DEBUG [RS_CLOSE_REGION-regionserver/1ab588ae992e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1732433728060.567d9cab2ff71509e601db9c0fa44478. 2024-11-24T07:35:52,818 INFO [RS:1;1ab588ae992e:40419 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-24T07:35:52,818 DEBUG [RS_CLOSE_REGION-regionserver/1ab588ae992e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1732433728060.567d9cab2ff71509e601db9c0fa44478. after waiting 0 ms 2024-11-24T07:35:52,818 INFO [RS:1;1ab588ae992e:40419 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-24T07:35:52,818 DEBUG [RS_CLOSE_REGION-regionserver/1ab588ae992e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsAfterRegionMovedWithMultiCF,,1732433728060.567d9cab2ff71509e601db9c0fa44478. 2024-11-24T07:35:52,818 INFO [RS:1;1ab588ae992e:40419 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-24T07:35:52,818 INFO [RS:1;1ab588ae992e:40419 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-11-24T07:35:52,818 INFO [RS:1;1ab588ae992e:40419 {}] regionserver.HRegionServer(1321): Waiting on 2 regions to close 2024-11-24T07:35:52,818 DEBUG [RS:1;1ab588ae992e:40419 {}] regionserver.HRegionServer(1325): Online Regions={1588230740=hbase:meta,,1.1588230740, 567d9cab2ff71509e601db9c0fa44478=testReplayEditsAfterRegionMovedWithMultiCF,,1732433728060.567d9cab2ff71509e601db9c0fa44478.} 2024-11-24T07:35:52,818 DEBUG [RS:1;1ab588ae992e:40419 {}] regionserver.HRegionServer(1351): Waiting on 1588230740, 567d9cab2ff71509e601db9c0fa44478 2024-11-24T07:35:52,818 DEBUG [RS_CLOSE_META-regionserver/1ab588ae992e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-24T07:35:52,818 INFO [RS_CLOSE_META-regionserver/1ab588ae992e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-24T07:35:52,818 DEBUG [RS_CLOSE_META-regionserver/1ab588ae992e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-24T07:35:52,819 DEBUG [RS_CLOSE_META-regionserver/1ab588ae992e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-24T07:35:52,819 DEBUG [RS_CLOSE_META-regionserver/1ab588ae992e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-24T07:35:52,819 INFO [RS_CLOSE_META-regionserver/1ab588ae992e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=1.19 KB heapSize=2.79 KB 2024-11-24T07:35:52,821 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41971 is added to blk_1073741834_1010 (size=1704) 2024-11-24T07:35:52,821 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41161 is added to blk_1073741834_1010 (size=1704) 2024-11-24T07:35:52,821 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42685 is added to blk_1073741834_1010 (size=1704) 2024-11-24T07:35:52,823 DEBUG [RS:0;1ab588ae992e:46433 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/e37915a4-8313-e328-91fb-37d541e8b214/oldWALs 2024-11-24T07:35:52,823 INFO [RS:0;1ab588ae992e:46433 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 1ab588ae992e%2C46433%2C1732433713473:(num 1732433715254) 2024-11-24T07:35:52,823 DEBUG [RS:0;1ab588ae992e:46433 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T07:35:52,823 INFO [RS:0;1ab588ae992e:46433 {}] regionserver.LeaseManager(133): Closed leases 2024-11-24T07:35:52,824 INFO [RS:0;1ab588ae992e:46433 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-24T07:35:52,824 INFO [RS:0;1ab588ae992e:46433 {}] hbase.ChoreService(370): Chore service for: regionserver/1ab588ae992e:0 had [ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-11-24T07:35:52,824 INFO [RS:0;1ab588ae992e:46433 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-24T07:35:52,824 INFO [regionserver/1ab588ae992e:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-24T07:35:52,824 INFO [RS:0;1ab588ae992e:46433 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-24T07:35:52,824 INFO [RS:0;1ab588ae992e:46433 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-24T07:35:52,824 INFO [RS:0;1ab588ae992e:46433 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-24T07:35:52,825 INFO [RS:0;1ab588ae992e:46433 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.3:46433 2024-11-24T07:35:52,826 DEBUG [RS_CLOSE_META-regionserver/1ab588ae992e:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33505/user/jenkins/test-data/e37915a4-8313-e328-91fb-37d541e8b214/data/hbase/meta/1588230740/.tmp/info/960706bb6dbc4aa9b586644158640e62 is 205, key is testReplayEditsAfterRegionMovedWithMultiCF,,1732433728060.567d9cab2ff71509e601db9c0fa44478./info:regioninfo/1732433732202/Put/seqid=0 2024-11-24T07:35:52,829 DEBUG [RS_CLOSE_REGION-regionserver/1ab588ae992e:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:33505/user/jenkins/test-data/e37915a4-8313-e328-91fb-37d541e8b214/data/default/testReplayEditsAfterRegionMovedWithMultiCF/567d9cab2ff71509e601db9c0fa44478/recovered.edits/20.seqid, newMaxSeqId=20, maxSeqId=17 2024-11-24T07:35:52,829 INFO [RS_CLOSE_REGION-regionserver/1ab588ae992e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testReplayEditsAfterRegionMovedWithMultiCF,,1732433728060.567d9cab2ff71509e601db9c0fa44478. 2024-11-24T07:35:52,829 DEBUG [RS_CLOSE_REGION-regionserver/1ab588ae992e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 567d9cab2ff71509e601db9c0fa44478: Waiting for close lock at 1732433752817Running coprocessor pre-close hooks at 1732433752817Disabling compacts and flushes for region at 1732433752817Disabling writes for close at 1732433752818 (+1 ms)Writing region close event to WAL at 1732433752823 (+5 ms)Running coprocessor post-close hooks at 1732433752829 (+6 ms)Closed at 1732433752829 2024-11-24T07:35:52,830 DEBUG [RS_CLOSE_REGION-regionserver/1ab588ae992e:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testReplayEditsAfterRegionMovedWithMultiCF,,1732433728060.567d9cab2ff71509e601db9c0fa44478. 2024-11-24T07:35:52,832 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41971 is added to blk_1073741943_1123 (size=6778) 2024-11-24T07:35:52,832 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42685 is added to blk_1073741943_1123 (size=6778) 2024-11-24T07:35:52,833 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41161 is added to blk_1073741943_1123 (size=6778) 2024-11-24T07:35:52,833 INFO [RS_CLOSE_META-regionserver/1ab588ae992e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.19 KB at sequenceid=23 (bloomFilter=true), to=hdfs://localhost:33505/user/jenkins/test-data/e37915a4-8313-e328-91fb-37d541e8b214/data/hbase/meta/1588230740/.tmp/info/960706bb6dbc4aa9b586644158640e62 2024-11-24T07:35:52,838 DEBUG [RS_CLOSE_META-regionserver/1ab588ae992e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33505/user/jenkins/test-data/e37915a4-8313-e328-91fb-37d541e8b214/data/hbase/meta/1588230740/.tmp/info/960706bb6dbc4aa9b586644158640e62 as hdfs://localhost:33505/user/jenkins/test-data/e37915a4-8313-e328-91fb-37d541e8b214/data/hbase/meta/1588230740/info/960706bb6dbc4aa9b586644158640e62 2024-11-24T07:35:52,843 INFO [RS_CLOSE_META-regionserver/1ab588ae992e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33505/user/jenkins/test-data/e37915a4-8313-e328-91fb-37d541e8b214/data/hbase/meta/1588230740/info/960706bb6dbc4aa9b586644158640e62, entries=8, sequenceid=23, filesize=6.6 K 2024-11-24T07:35:52,844 INFO [RS_CLOSE_META-regionserver/1ab588ae992e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~1.19 KB/1218, heapSize ~2.02 KB/2072, currentSize=0 B/0 for 1588230740 in 25ms, sequenceid=23, compaction requested=false 2024-11-24T07:35:52,848 DEBUG [RS_CLOSE_META-regionserver/1ab588ae992e:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:33505/user/jenkins/test-data/e37915a4-8313-e328-91fb-37d541e8b214/data/hbase/meta/1588230740/recovered.edits/26.seqid, newMaxSeqId=26, maxSeqId=18 2024-11-24T07:35:52,849 DEBUG [RS_CLOSE_META-regionserver/1ab588ae992e:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-24T07:35:52,849 INFO [RS_CLOSE_META-regionserver/1ab588ae992e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-24T07:35:52,849 DEBUG [RS_CLOSE_META-regionserver/1ab588ae992e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1732433752818Running coprocessor pre-close hooks at 1732433752818Disabling compacts and flushes for region at 1732433752818Disabling writes for close at 1732433752819 (+1 ms)Obtaining lock to block concurrent updates at 1732433752819Preparing flush snapshotting stores in 1588230740 at 1732433752819Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=1218, getHeapSize=2792, getOffHeapSize=0, getCellsCount=8 at 1732433752819Flushing stores of hbase:meta,,1.1588230740 at 1732433752820 (+1 ms)Flushing 1588230740/info: creating writer at 1732433752820Flushing 1588230740/info: appending metadata at 1732433752825 (+5 ms)Flushing 1588230740/info: closing flushed file at 1732433752825Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@5f613a50: reopening flushed file at 1732433752837 (+12 ms)Finished flush of dataSize ~1.19 KB/1218, heapSize ~2.02 KB/2072, currentSize=0 B/0 for 1588230740 in 25ms, sequenceid=23, compaction requested=false at 1732433752844 (+7 ms)Writing region close event to WAL at 1732433752845 (+1 ms)Running coprocessor post-close hooks at 1732433752849 (+4 ms)Closed at 1732433752849 2024-11-24T07:35:52,849 DEBUG [RS_CLOSE_META-regionserver/1ab588ae992e:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-11-24T07:35:52,856 INFO [regionserver/1ab588ae992e:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-24T07:35:52,857 INFO [regionserver/1ab588ae992e:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-24T07:35:52,875 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37719-0x1016bc48eea0000, quorum=127.0.0.1:49244, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-24T07:35:52,875 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46433-0x1016bc48eea0001, quorum=127.0.0.1:49244, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/1ab588ae992e,46433,1732433713473 2024-11-24T07:35:52,875 INFO [RS:0;1ab588ae992e:46433 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-24T07:35:52,876 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [1ab588ae992e,46433,1732433713473] 2024-11-24T07:35:52,907 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/1ab588ae992e,46433,1732433713473 already deleted, retry=false 2024-11-24T07:35:52,907 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; 1ab588ae992e,46433,1732433713473 expired; onlineServers=1 2024-11-24T07:35:52,996 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46433-0x1016bc48eea0001, quorum=127.0.0.1:49244, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-24T07:35:52,997 INFO [RS:0;1ab588ae992e:46433 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-24T07:35:52,997 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46433-0x1016bc48eea0001, quorum=127.0.0.1:49244, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-24T07:35:52,997 INFO [RS:0;1ab588ae992e:46433 {}] regionserver.HRegionServer(1031): Exiting; stopping=1ab588ae992e,46433,1732433713473; zookeeper connection closed. 2024-11-24T07:35:52,998 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@2c265f05 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@2c265f05 2024-11-24T07:35:53,019 INFO [RS:1;1ab588ae992e:40419 {}] regionserver.HRegionServer(976): stopping server 1ab588ae992e,40419,1732433713579; all regions closed. 2024-11-24T07:35:53,025 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41971 is added to blk_1073741893_1071 (size=1670) 2024-11-24T07:35:53,026 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41161 is added to blk_1073741893_1071 (size=1670) 2024-11-24T07:35:53,026 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42685 is added to blk_1073741893_1071 (size=1670) 2024-11-24T07:35:53,029 DEBUG [RS:1;1ab588ae992e:40419 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/e37915a4-8313-e328-91fb-37d541e8b214/oldWALs 2024-11-24T07:35:53,029 INFO [RS:1;1ab588ae992e:40419 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 1ab588ae992e%2C40419%2C1732433713579.meta:.meta(num 1732433731328) 2024-11-24T07:35:53,032 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41161 is added to blk_1073741833_1009 (size=719) 2024-11-24T07:35:53,032 WARN [Close-WAL-Writer-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(650): complete file /user/jenkins/test-data/e37915a4-8313-e328-91fb-37d541e8b214/WALs/1ab588ae992e,40419,1732433713579/1ab588ae992e%2C40419%2C1732433713579.1732433715254 not finished, retry = 0 2024-11-24T07:35:53,033 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41971 is added to blk_1073741833_1009 (size=719) 2024-11-24T07:35:53,033 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42685 is added to blk_1073741833_1009 (size=719) 2024-11-24T07:35:53,050 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testDatalossWhenInputError 2024-11-24T07:35:53,051 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testDatalossWhenInputError Metrics about Tables on a single HBase RegionServer 2024-11-24T07:35:53,051 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testCompactedBulkLoadedFiles 2024-11-24T07:35:53,051 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testCompactedBulkLoadedFiles Metrics about Tables on a single HBase RegionServer 2024-11-24T07:35:53,052 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testReplayEditsAfterAbortingFlush 2024-11-24T07:35:53,052 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testReplayEditsAfterAbortingFlush Metrics about Tables on a single HBase RegionServer 2024-11-24T07:35:53,052 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-24T07:35:53,052 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta 2024-11-24T07:35:53,052 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testReplayEditsAfterRegionMovedWithMultiCF 2024-11-24T07:35:53,105 INFO [regionserver/1ab588ae992e:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: CompactionChecker was stopped 2024-11-24T07:35:53,105 INFO [regionserver/1ab588ae992e:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: MemstoreFlusherChore was stopped 2024-11-24T07:35:53,138 DEBUG [RS:1;1ab588ae992e:40419 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/e37915a4-8313-e328-91fb-37d541e8b214/oldWALs 2024-11-24T07:35:53,138 INFO [RS:1;1ab588ae992e:40419 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 1ab588ae992e%2C40419%2C1732433713579:(num 1732433715254) 2024-11-24T07:35:53,138 DEBUG [RS:1;1ab588ae992e:40419 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T07:35:53,138 INFO [RS:1;1ab588ae992e:40419 {}] regionserver.LeaseManager(133): Closed leases 2024-11-24T07:35:53,138 INFO [RS:1;1ab588ae992e:40419 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-24T07:35:53,139 INFO [RS:1;1ab588ae992e:40419 {}] hbase.ChoreService(370): Chore service for: regionserver/1ab588ae992e:0 had [ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS] on shutdown 2024-11-24T07:35:53,139 INFO [RS:1;1ab588ae992e:40419 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-24T07:35:53,139 INFO [regionserver/1ab588ae992e:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-24T07:35:53,139 INFO [RS:1;1ab588ae992e:40419 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.3:40419 2024-11-24T07:35:53,156 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40419-0x1016bc48eea0002, quorum=127.0.0.1:49244, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/1ab588ae992e,40419,1732433713579 2024-11-24T07:35:53,156 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37719-0x1016bc48eea0000, quorum=127.0.0.1:49244, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-24T07:35:53,157 INFO [RS:1;1ab588ae992e:40419 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-24T07:35:53,170 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [1ab588ae992e,40419,1732433713579] 2024-11-24T07:35:53,180 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/1ab588ae992e,40419,1732433713579 already deleted, retry=false 2024-11-24T07:35:53,180 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; 1ab588ae992e,40419,1732433713579 expired; onlineServers=0 2024-11-24T07:35:53,180 INFO [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master '1ab588ae992e,37719,1732433712713' ***** 2024-11-24T07:35:53,180 INFO [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0 2024-11-24T07:35:53,180 INFO [M:0;1ab588ae992e:37719 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-24T07:35:53,180 INFO [M:0;1ab588ae992e:37719 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-24T07:35:53,180 DEBUG [M:0;1ab588ae992e:37719 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2024-11-24T07:35:53,181 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2024-11-24T07:35:53,181 DEBUG [M:0;1ab588ae992e:37719 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2024-11-24T07:35:53,181 DEBUG [master/1ab588ae992e:0:becomeActiveMaster-HFileCleaner.small.0-1732433714923 {}] cleaner.HFileCleaner(306): Exit Thread[master/1ab588ae992e:0:becomeActiveMaster-HFileCleaner.small.0-1732433714923,5,FailOnTimeoutGroup] 2024-11-24T07:35:53,181 DEBUG [master/1ab588ae992e:0:becomeActiveMaster-HFileCleaner.large.0-1732433714922 {}] cleaner.HFileCleaner(306): Exit Thread[master/1ab588ae992e:0:becomeActiveMaster-HFileCleaner.large.0-1732433714922,5,FailOnTimeoutGroup] 2024-11-24T07:35:53,181 INFO [M:0;1ab588ae992e:37719 {}] hbase.ChoreService(370): Chore service for: master/1ab588ae992e:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown 2024-11-24T07:35:53,181 INFO [M:0;1ab588ae992e:37719 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-24T07:35:53,181 DEBUG [M:0;1ab588ae992e:37719 {}] master.HMaster(1795): Stopping service threads 2024-11-24T07:35:53,181 INFO [M:0;1ab588ae992e:37719 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2024-11-24T07:35:53,181 INFO [M:0;1ab588ae992e:37719 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-24T07:35:53,182 INFO [M:0;1ab588ae992e:37719 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-24T07:35:53,182 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2024-11-24T07:35:53,191 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37719-0x1016bc48eea0000, quorum=127.0.0.1:49244, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2024-11-24T07:35:53,191 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37719-0x1016bc48eea0000, quorum=127.0.0.1:49244, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-24T07:35:53,191 DEBUG [M:0;1ab588ae992e:37719 {}] zookeeper.ZKUtil(347): master:37719-0x1016bc48eea0000, quorum=127.0.0.1:49244, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error) 2024-11-24T07:35:53,191 WARN [M:0;1ab588ae992e:37719 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null 2024-11-24T07:35:53,192 INFO [M:0;1ab588ae992e:37719 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: hdfs://localhost:33505/user/jenkins/test-data/e37915a4-8313-e328-91fb-37d541e8b214/.lastflushedseqids 2024-11-24T07:35:53,207 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41161 is added to blk_1073741944_1124 (size=119) 2024-11-24T07:35:53,207 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41971 is added to blk_1073741944_1124 (size=119) 2024-11-24T07:35:53,207 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42685 is added to blk_1073741944_1124 (size=119) 2024-11-24T07:35:53,209 INFO [M:0;1ab588ae992e:37719 {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-24T07:35:53,209 INFO [M:0;1ab588ae992e:37719 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false 2024-11-24T07:35:53,209 DEBUG [M:0;1ab588ae992e:37719 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-24T07:35:53,209 INFO [M:0;1ab588ae992e:37719 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-24T07:35:53,209 DEBUG [M:0;1ab588ae992e:37719 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-24T07:35:53,209 DEBUG [M:0;1ab588ae992e:37719 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-24T07:35:53,209 DEBUG [M:0;1ab588ae992e:37719 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-24T07:35:53,209 INFO [M:0;1ab588ae992e:37719 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=83.40 KB heapSize=102.70 KB 2024-11-24T07:35:53,226 DEBUG [M:0;1ab588ae992e:37719 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33505/user/jenkins/test-data/e37915a4-8313-e328-91fb-37d541e8b214/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/7908f6a024464f879445e2145c20bded is 82, key is hbase:meta,,1/info:regioninfo/1732433731481/Put/seqid=0 2024-11-24T07:35:53,231 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41971 is added to blk_1073741945_1125 (size=6063) 2024-11-24T07:35:53,232 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42685 is added to blk_1073741945_1125 (size=6063) 2024-11-24T07:35:53,232 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41161 is added to blk_1073741945_1125 (size=6063) 2024-11-24T07:35:53,232 INFO [M:0;1ab588ae992e:37719 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1008 B at sequenceid=207 (bloomFilter=true), to=hdfs://localhost:33505/user/jenkins/test-data/e37915a4-8313-e328-91fb-37d541e8b214/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/7908f6a024464f879445e2145c20bded 2024-11-24T07:35:53,251 DEBUG [M:0;1ab588ae992e:37719 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33505/user/jenkins/test-data/e37915a4-8313-e328-91fb-37d541e8b214/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/c950ef0ea25e4ceeb4a95026a9c5c87f is 1076, key is \x00\x00\x00\x00\x00\x00\x00\x04/proc:d/1732433728489/Put/seqid=0 2024-11-24T07:35:53,257 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42685 is added to blk_1073741946_1126 (size=7907) 2024-11-24T07:35:53,257 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41971 is added to blk_1073741946_1126 (size=7907) 2024-11-24T07:35:53,257 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41161 is added to blk_1073741946_1126 (size=7907) 2024-11-24T07:35:53,258 INFO [M:0;1ab588ae992e:37719 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=82.17 KB at sequenceid=207 (bloomFilter=true), to=hdfs://localhost:33505/user/jenkins/test-data/e37915a4-8313-e328-91fb-37d541e8b214/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/c950ef0ea25e4ceeb4a95026a9c5c87f 2024-11-24T07:35:53,262 INFO [M:0;1ab588ae992e:37719 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for c950ef0ea25e4ceeb4a95026a9c5c87f 2024-11-24T07:35:53,270 INFO [RS:1;1ab588ae992e:40419 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-24T07:35:53,270 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40419-0x1016bc48eea0002, quorum=127.0.0.1:49244, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-24T07:35:53,270 INFO [RS:1;1ab588ae992e:40419 {}] regionserver.HRegionServer(1031): Exiting; stopping=1ab588ae992e,40419,1732433713579; zookeeper connection closed. 2024-11-24T07:35:53,270 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40419-0x1016bc48eea0002, quorum=127.0.0.1:49244, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-24T07:35:53,270 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@4f9651ae {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@4f9651ae 2024-11-24T07:35:53,271 INFO [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 3 regionserver(s) complete 2024-11-24T07:35:53,276 DEBUG [M:0;1ab588ae992e:37719 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33505/user/jenkins/test-data/e37915a4-8313-e328-91fb-37d541e8b214/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/7c04567b24bc473db04103ae11c02426 is 69, key is 1ab588ae992e,40419,1732433713579/rs:state/1732433714954/Put/seqid=0 2024-11-24T07:35:53,281 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41161 is added to blk_1073741947_1127 (size=5445) 2024-11-24T07:35:53,282 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42685 is added to blk_1073741947_1127 (size=5445) 2024-11-24T07:35:53,282 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41971 is added to blk_1073741947_1127 (size=5445) 2024-11-24T07:35:53,282 INFO [M:0;1ab588ae992e:37719 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=249 B at sequenceid=207 (bloomFilter=true), to=hdfs://localhost:33505/user/jenkins/test-data/e37915a4-8313-e328-91fb-37d541e8b214/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/7c04567b24bc473db04103ae11c02426 2024-11-24T07:35:53,287 INFO [M:0;1ab588ae992e:37719 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 7c04567b24bc473db04103ae11c02426 2024-11-24T07:35:53,288 DEBUG [M:0;1ab588ae992e:37719 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33505/user/jenkins/test-data/e37915a4-8313-e328-91fb-37d541e8b214/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/7908f6a024464f879445e2145c20bded as hdfs://localhost:33505/user/jenkins/test-data/e37915a4-8313-e328-91fb-37d541e8b214/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/7908f6a024464f879445e2145c20bded 2024-11-24T07:35:53,294 INFO [M:0;1ab588ae992e:37719 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33505/user/jenkins/test-data/e37915a4-8313-e328-91fb-37d541e8b214/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/7908f6a024464f879445e2145c20bded, entries=14, sequenceid=207, filesize=5.9 K 2024-11-24T07:35:53,295 DEBUG [M:0;1ab588ae992e:37719 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33505/user/jenkins/test-data/e37915a4-8313-e328-91fb-37d541e8b214/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/c950ef0ea25e4ceeb4a95026a9c5c87f as hdfs://localhost:33505/user/jenkins/test-data/e37915a4-8313-e328-91fb-37d541e8b214/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/c950ef0ea25e4ceeb4a95026a9c5c87f 2024-11-24T07:35:53,301 INFO [M:0;1ab588ae992e:37719 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for c950ef0ea25e4ceeb4a95026a9c5c87f 2024-11-24T07:35:53,301 INFO [M:0;1ab588ae992e:37719 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33505/user/jenkins/test-data/e37915a4-8313-e328-91fb-37d541e8b214/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/c950ef0ea25e4ceeb4a95026a9c5c87f, entries=21, sequenceid=207, filesize=7.7 K 2024-11-24T07:35:53,302 DEBUG [M:0;1ab588ae992e:37719 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33505/user/jenkins/test-data/e37915a4-8313-e328-91fb-37d541e8b214/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/7c04567b24bc473db04103ae11c02426 as hdfs://localhost:33505/user/jenkins/test-data/e37915a4-8313-e328-91fb-37d541e8b214/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/7c04567b24bc473db04103ae11c02426 2024-11-24T07:35:53,307 INFO [M:0;1ab588ae992e:37719 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 7c04567b24bc473db04103ae11c02426 2024-11-24T07:35:53,307 INFO [M:0;1ab588ae992e:37719 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33505/user/jenkins/test-data/e37915a4-8313-e328-91fb-37d541e8b214/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/7c04567b24bc473db04103ae11c02426, entries=3, sequenceid=207, filesize=5.3 K 2024-11-24T07:35:53,308 INFO [M:0;1ab588ae992e:37719 {}] regionserver.HRegion(3140): Finished flush of dataSize ~83.40 KB/85398, heapSize ~102.41 KB/104864, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 99ms, sequenceid=207, compaction requested=false 2024-11-24T07:35:53,309 INFO [M:0;1ab588ae992e:37719 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-24T07:35:53,309 DEBUG [M:0;1ab588ae992e:37719 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1732433753209Disabling compacts and flushes for region at 1732433753209Disabling writes for close at 1732433753209Obtaining lock to block concurrent updates at 1732433753209Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1732433753209Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=85398, getHeapSize=105104, getOffHeapSize=0, getCellsCount=248 at 1732433753209Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1732433753210 (+1 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1732433753210Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1732433753226 (+16 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1732433753226Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1732433753236 (+10 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1732433753251 (+15 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1732433753251Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1732433753262 (+11 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1732433753276 (+14 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1732433753276Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@3ad267d2: reopening flushed file at 1732433753287 (+11 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@3c2de001: reopening flushed file at 1732433753294 (+7 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@60019d76: reopening flushed file at 1732433753301 (+7 ms)Finished flush of dataSize ~83.40 KB/85398, heapSize ~102.41 KB/104864, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 99ms, sequenceid=207, compaction requested=false at 1732433753308 (+7 ms)Writing region close event to WAL at 1732433753309 (+1 ms)Closed at 1732433753309 2024-11-24T07:35:53,312 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42685 is added to blk_1073741830_1006 (size=69324) 2024-11-24T07:35:53,312 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41161 is added to blk_1073741830_1006 (size=69324) 2024-11-24T07:35:53,312 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41971 is added to blk_1073741830_1006 (size=69324) 2024-11-24T07:35:53,313 INFO [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-24T07:35:53,313 INFO [M:0;1ab588ae992e:37719 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down. 2024-11-24T07:35:53,313 INFO [M:0;1ab588ae992e:37719 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.3:37719 2024-11-24T07:35:53,313 INFO [M:0;1ab588ae992e:37719 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-24T07:35:53,425 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37719-0x1016bc48eea0000, quorum=127.0.0.1:49244, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-24T07:35:53,425 INFO [M:0;1ab588ae992e:37719 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-24T07:35:53,425 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37719-0x1016bc48eea0000, quorum=127.0.0.1:49244, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-24T07:35:53,436 ERROR [Time-limited test {}] hdfs.DFSClient(665): Failed to close file: /hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1732433733272/wal.1732433742937 with renewLeaseKey: DEFAULT_16688 java.io.FileNotFoundException: File does not exist: /hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1732433733272/wal.1732433742937 (inode 16688) Holder DFSClient_NONMAPREDUCE_-166876350_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.ipc.RemoteException.instantiateException(RemoteException.java:121) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.RemoteException.unwrapRemoteException(RemoteException.java:110) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:653) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.close(FanOutOneBlockAsyncDFSOutput.java:619) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hdfs.DummyDFSOutputStream.close(DummyDFSOutputStream.java:52) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeAllFilesBeingWritten(DFSClient.java:662) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeOutputStreams(DFSClient.java:699) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.close(DistributedFileSystem.java:1528) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2194) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2174) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2167) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniDFSCluster(HBaseTestingUtil.java:761) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1021) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.ipc.RemoteException: File does not exist: /hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1732433733272/wal.1732433742937 (inode 16688) Holder DFSClient_NONMAPREDUCE_-166876350_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1584) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy43.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.lambda$complete$13(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.complete(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:437) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:170) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:162) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:100) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:366) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy44.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:646) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 25 more 2024-11-24T07:35:53,439 ERROR [Time-limited test {}] hdfs.DFSClient(665): Failed to close file: /hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1732433732714/wal.1732433733108 with renewLeaseKey: DEFAULT_16665 java.io.FileNotFoundException: File does not exist: /hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1732433732714/wal.1732433733108 (inode 16665) Holder DFSClient_NONMAPREDUCE_-166876350_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.ipc.RemoteException.instantiateException(RemoteException.java:121) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.RemoteException.unwrapRemoteException(RemoteException.java:110) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:653) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.close(FanOutOneBlockAsyncDFSOutput.java:619) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hdfs.DummyDFSOutputStream.close(DummyDFSOutputStream.java:52) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeAllFilesBeingWritten(DFSClient.java:662) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeOutputStreams(DFSClient.java:699) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.close(DistributedFileSystem.java:1528) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2194) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2174) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2167) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniDFSCluster(HBaseTestingUtil.java:761) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1021) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.ipc.RemoteException: File does not exist: /hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1732433732714/wal.1732433733108 (inode 16665) Holder DFSClient_NONMAPREDUCE_-166876350_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1584) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy43.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.lambda$complete$13(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.complete(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:437) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:170) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:162) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:100) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:366) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy44.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:646) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 25 more 2024-11-24T07:35:53,439 ERROR [Time-limited test {}] hdfs.DFSClient(665): Failed to close file: /hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1732433723663/wal.1732433723759 with renewLeaseKey: DEFAULT_16586 java.io.IOException: stream already broken at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.endBlock(FanOutOneBlockAsyncDFSOutput.java:566) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.close(FanOutOneBlockAsyncDFSOutput.java:615) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hdfs.DummyDFSOutputStream.close(DummyDFSOutputStream.java:52) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeAllFilesBeingWritten(DFSClient.java:662) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeOutputStreams(DFSClient.java:699) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.close(DistributedFileSystem.java:1528) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2194) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2174) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2167) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniDFSCluster(HBaseTestingUtil.java:761) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1021) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T07:35:53,441 ERROR [Time-limited test {}] hdfs.DFSClient(665): Failed to close file: /hbase/WALs/testdatalosswheninputerror-manual,16010,1732433743141/wal.1732433743641 with renewLeaseKey: DEFAULT_16714 java.io.FileNotFoundException: File does not exist: /hbase/WALs/testdatalosswheninputerror-manual,16010,1732433743141/wal.1732433743641 (inode 16714) Holder DFSClient_NONMAPREDUCE_-166876350_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.ipc.RemoteException.instantiateException(RemoteException.java:121) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.RemoteException.unwrapRemoteException(RemoteException.java:110) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:653) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.close(FanOutOneBlockAsyncDFSOutput.java:619) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hdfs.DummyDFSOutputStream.close(DummyDFSOutputStream.java:52) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeAllFilesBeingWritten(DFSClient.java:662) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeOutputStreams(DFSClient.java:699) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.close(DistributedFileSystem.java:1528) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2194) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2174) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2167) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniDFSCluster(HBaseTestingUtil.java:761) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1021) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.ipc.RemoteException: File does not exist: /hbase/WALs/testdatalosswheninputerror-manual,16010,1732433743141/wal.1732433743641 (inode 16714) Holder DFSClient_NONMAPREDUCE_-166876350_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1584) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy43.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.lambda$complete$13(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.complete(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:437) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:170) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:162) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:100) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:366) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy44.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:646) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 25 more 2024-11-24T07:35:53,442 ERROR [Time-limited test {}] hdfs.DFSClient(665): Failed to close file: /hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1732433743778/wal.1732433743833 with renewLeaseKey: DEFAULT_16736 java.io.IOException: stream already broken at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.endBlock(FanOutOneBlockAsyncDFSOutput.java:566) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.close(FanOutOneBlockAsyncDFSOutput.java:615) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hdfs.DummyDFSOutputStream.close(DummyDFSOutputStream.java:52) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeAllFilesBeingWritten(DFSClient.java:662) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeOutputStreams(DFSClient.java:699) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.close(DistributedFileSystem.java:1528) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2194) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2174) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2167) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniDFSCluster(HBaseTestingUtil.java:761) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1021) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T07:35:53,442 ERROR [Time-limited test {}] hdfs.DFSClient(665): Failed to close file: /hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732433748177/wal.1732433748351 with renewLeaseKey: DEFAULT_16777 java.io.IOException: stream already broken at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.endBlock(FanOutOneBlockAsyncDFSOutput.java:566) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.close(FanOutOneBlockAsyncDFSOutput.java:615) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hdfs.DummyDFSOutputStream.close(DummyDFSOutputStream.java:52) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeAllFilesBeingWritten(DFSClient.java:662) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeOutputStreams(DFSClient.java:699) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.close(DistributedFileSystem.java:1528) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2194) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2174) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2167) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniDFSCluster(HBaseTestingUtil.java:761) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1021) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T07:35:53,443 ERROR [Time-limited test {}] hdfs.DFSClient(665): Failed to close file: /hbase/WALs/testreplayeditswrittenintowal-manual,16010,1732433716991/wal.1732433717055 with renewLeaseKey: DEFAULT_16506 java.io.IOException: stream already broken at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.endBlock(FanOutOneBlockAsyncDFSOutput.java:566) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.close(FanOutOneBlockAsyncDFSOutput.java:615) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hdfs.DummyDFSOutputStream.close(DummyDFSOutputStream.java:52) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeAllFilesBeingWritten(DFSClient.java:662) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeOutputStreams(DFSClient.java:699) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.close(DistributedFileSystem.java:1528) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2194) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2174) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2167) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniDFSCluster(HBaseTestingUtil.java:761) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1021) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T07:35:53,444 ERROR [Time-limited test {}] hdfs.DFSClient(665): Failed to close file: /hbase/WALs/testnameconflictwhensplit1-manual,16010,1732433716657/wal.1732433716853 with renewLeaseKey: DEFAULT_16485 java.io.FileNotFoundException: File does not exist: /hbase/WALs/testnameconflictwhensplit1-manual,16010,1732433716657/wal.1732433716853 (inode 16485) Holder DFSClient_NONMAPREDUCE_-166876350_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.ipc.RemoteException.instantiateException(RemoteException.java:121) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.RemoteException.unwrapRemoteException(RemoteException.java:110) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:653) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.close(FanOutOneBlockAsyncDFSOutput.java:619) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hdfs.DummyDFSOutputStream.close(DummyDFSOutputStream.java:52) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeAllFilesBeingWritten(DFSClient.java:662) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeOutputStreams(DFSClient.java:699) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.close(DistributedFileSystem.java:1528) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2194) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2174) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2167) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniDFSCluster(HBaseTestingUtil.java:761) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1021) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.ipc.RemoteException: File does not exist: /hbase/WALs/testnameconflictwhensplit1-manual,16010,1732433716657/wal.1732433716853 (inode 16485) Holder DFSClient_NONMAPREDUCE_-166876350_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1584) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy43.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.lambda$complete$13(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.complete(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:437) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:170) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:162) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:100) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:366) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy44.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:646) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 25 more 2024-11-24T07:35:53,447 ERROR [Time-limited test {}] hdfs.DFSClient(665): Failed to close file: /hbase/WALs/testnameconflictwhensplit0-manual,16010,1732433716176/wal.1732433716419 with renewLeaseKey: DEFAULT_16462 java.io.FileNotFoundException: File does not exist: /hbase/WALs/testnameconflictwhensplit0-manual,16010,1732433716176/wal.1732433716419 (inode 16462) Holder DFSClient_NONMAPREDUCE_-166876350_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.ipc.RemoteException.instantiateException(RemoteException.java:121) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.RemoteException.unwrapRemoteException(RemoteException.java:110) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:653) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.close(FanOutOneBlockAsyncDFSOutput.java:619) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hdfs.DummyDFSOutputStream.close(DummyDFSOutputStream.java:52) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeAllFilesBeingWritten(DFSClient.java:662) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeOutputStreams(DFSClient.java:699) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.close(DistributedFileSystem.java:1528) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2194) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2174) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2167) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniDFSCluster(HBaseTestingUtil.java:761) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1021) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.ipc.RemoteException: File does not exist: /hbase/WALs/testnameconflictwhensplit0-manual,16010,1732433716176/wal.1732433716419 (inode 16462) Holder DFSClient_NONMAPREDUCE_-166876350_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1584) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy43.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.lambda$complete$13(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.complete(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:437) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:170) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:162) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:100) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:366) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy44.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:646) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 25 more 2024-11-24T07:35:53,451 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@35f1150e{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-24T07:35:53,453 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@13a77e13{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-24T07:35:53,454 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-24T07:35:53,454 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@f9972d0{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-24T07:35:53,454 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@2a6d5e13{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/089ee269-8889-d2f6-f8a2-4bb8eac365fc/hadoop.log.dir/,STOPPED} 2024-11-24T07:35:53,457 WARN [BP-1337235608-172.17.0.3-1732433708251 heartbeating to localhost/127.0.0.1:33505 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-11-24T07:35:53,457 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-11-24T07:35:53,457 WARN [BP-1337235608-172.17.0.3-1732433708251 heartbeating to localhost/127.0.0.1:33505 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-1337235608-172.17.0.3-1732433708251 (Datanode Uuid 3e29c02d-96eb-4f29-acd8-ca7a6a190670) service to localhost/127.0.0.1:33505 2024-11-24T07:35:53,457 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-11-24T07:35:53,458 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/089ee269-8889-d2f6-f8a2-4bb8eac365fc/cluster_a7d76481-495d-c9f6-361f-c7016573445a/data/data5/current/BP-1337235608-172.17.0.3-1732433708251 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-24T07:35:53,458 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/089ee269-8889-d2f6-f8a2-4bb8eac365fc/cluster_a7d76481-495d-c9f6-361f-c7016573445a/data/data6/current/BP-1337235608-172.17.0.3-1732433708251 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-24T07:35:53,459 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-11-24T07:35:53,462 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@7bd427b8{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-24T07:35:53,462 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@6915083f{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-24T07:35:53,462 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-24T07:35:53,462 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@5cc2d6b{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-24T07:35:53,462 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@46b092e1{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/089ee269-8889-d2f6-f8a2-4bb8eac365fc/hadoop.log.dir/,STOPPED} 2024-11-24T07:35:53,463 WARN [BP-1337235608-172.17.0.3-1732433708251 heartbeating to localhost/127.0.0.1:33505 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-11-24T07:35:53,464 WARN [BP-1337235608-172.17.0.3-1732433708251 heartbeating to localhost/127.0.0.1:33505 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-1337235608-172.17.0.3-1732433708251 (Datanode Uuid 91e56bd8-6eba-48c1-9801-3407dc9994e7) service to localhost/127.0.0.1:33505 2024-11-24T07:35:53,464 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-11-24T07:35:53,464 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-11-24T07:35:53,464 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/089ee269-8889-d2f6-f8a2-4bb8eac365fc/cluster_a7d76481-495d-c9f6-361f-c7016573445a/data/data3/current/BP-1337235608-172.17.0.3-1732433708251 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-24T07:35:53,465 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/089ee269-8889-d2f6-f8a2-4bb8eac365fc/cluster_a7d76481-495d-c9f6-361f-c7016573445a/data/data4/current/BP-1337235608-172.17.0.3-1732433708251 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-24T07:35:53,465 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-11-24T07:35:53,471 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@330740de{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-24T07:35:53,471 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@7b24cab9{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-24T07:35:53,471 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-24T07:35:53,472 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@4a359997{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-24T07:35:53,472 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@cf5a85e{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/089ee269-8889-d2f6-f8a2-4bb8eac365fc/hadoop.log.dir/,STOPPED} 2024-11-24T07:35:53,473 WARN [BP-1337235608-172.17.0.3-1732433708251 heartbeating to localhost/127.0.0.1:33505 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-11-24T07:35:53,473 WARN [BP-1337235608-172.17.0.3-1732433708251 heartbeating to localhost/127.0.0.1:33505 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-1337235608-172.17.0.3-1732433708251 (Datanode Uuid 36f029f0-d600-4005-8031-fb9832321be4) service to localhost/127.0.0.1:33505 2024-11-24T07:35:53,473 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-11-24T07:35:53,473 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-11-24T07:35:53,473 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/089ee269-8889-d2f6-f8a2-4bb8eac365fc/cluster_a7d76481-495d-c9f6-361f-c7016573445a/data/data1/current/BP-1337235608-172.17.0.3-1732433708251 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-24T07:35:53,474 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/089ee269-8889-d2f6-f8a2-4bb8eac365fc/cluster_a7d76481-495d-c9f6-361f-c7016573445a/data/data2/current/BP-1337235608-172.17.0.3-1732433708251 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-24T07:35:53,474 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-11-24T07:35:53,479 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@3717288f{hdfs,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-11-24T07:35:53,480 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@4bd70930{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-24T07:35:53,480 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-24T07:35:53,480 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@6dc9d5c1{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-24T07:35:53,480 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@4f37ffca{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/089ee269-8889-d2f6-f8a2-4bb8eac365fc/hadoop.log.dir/,STOPPED} 2024-11-24T07:35:53,489 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers 2024-11-24T07:35:53,538 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down