2024-11-24 08:29:32,429 main DEBUG Apache Log4j Core 2.17.2 initializing configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@20b2475a 2024-11-24 08:29:32,441 main DEBUG Took 0.009653 seconds to load 1 plugins from package org.apache.hadoop.hbase.logging 2024-11-24 08:29:32,441 main DEBUG PluginManager 'Core' found 129 plugins 2024-11-24 08:29:32,442 main DEBUG PluginManager 'Level' found 0 plugins 2024-11-24 08:29:32,442 main DEBUG PluginManager 'Lookup' found 16 plugins 2024-11-24 08:29:32,444 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-24 08:29:32,450 main DEBUG PluginManager 'TypeConverter' found 26 plugins 2024-11-24 08:29:32,462 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.util.MBeans", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-24 08:29:32,463 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-24 08:29:32,464 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.logging.TestJul2Slf4j", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-24 08:29:32,464 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-24 08:29:32,464 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.zookeeper", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-24 08:29:32,465 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-24 08:29:32,465 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSinkAdapter", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-24 08:29:32,466 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-24 08:29:32,466 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSystemImpl", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-24 08:29:32,466 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-24 08:29:32,467 main DEBUG LoggerConfig$Builder(additivity="false", level="WARN", levelAndRefs="null", name="org.apache.directory", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-24 08:29:32,467 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-24 08:29:32,468 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.ipc.FailedServers", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-24 08:29:32,468 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-24 08:29:32,468 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsConfig", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-24 08:29:32,469 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-24 08:29:32,469 main DEBUG LoggerConfig$Builder(additivity="null", level="INFO", levelAndRefs="null", name="org.apache.hadoop.hbase.ScheduledChore", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-24 08:29:32,469 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-24 08:29:32,470 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.regionserver.RSRpcServices", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-24 08:29:32,470 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-24 08:29:32,470 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-24 08:29:32,471 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-24 08:29:32,471 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-24 08:29:32,471 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-24 08:29:32,472 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hbase.thirdparty.io.netty.channel", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-24 08:29:32,472 main DEBUG Building Plugin[name=root, class=org.apache.logging.log4j.core.config.LoggerConfig$RootLogger]. 2024-11-24 08:29:32,473 main DEBUG LoggerConfig$RootLogger$Builder(additivity="null", level="null", levelAndRefs="INFO,Console", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-24 08:29:32,474 main DEBUG Building Plugin[name=loggers, class=org.apache.logging.log4j.core.config.LoggersPlugin]. 2024-11-24 08:29:32,476 main DEBUG createLoggers(={org.apache.hadoop.metrics2.util.MBeans, org.apache.hadoop.hbase.logging.TestJul2Slf4j, org.apache.zookeeper, org.apache.hadoop.metrics2.impl.MetricsSinkAdapter, org.apache.hadoop.metrics2.impl.MetricsSystemImpl, org.apache.directory, org.apache.hadoop.hbase.ipc.FailedServers, org.apache.hadoop.metrics2.impl.MetricsConfig, org.apache.hadoop.hbase.ScheduledChore, org.apache.hadoop.hbase.regionserver.RSRpcServices, org.apache.hadoop, org.apache.hadoop.hbase, org.apache.hbase.thirdparty.io.netty.channel, root}) 2024-11-24 08:29:32,476 main DEBUG Building Plugin[name=layout, class=org.apache.logging.log4j.core.layout.PatternLayout]. 2024-11-24 08:29:32,477 main DEBUG PatternLayout$Builder(pattern="%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n", PatternSelector=null, Configuration(PropertiesConfig), Replace=null, charset="null", alwaysWriteExceptions="null", disableAnsi="null", noConsoleNoAnsi="null", header="null", footer="null") 2024-11-24 08:29:32,478 main DEBUG PluginManager 'Converter' found 47 plugins 2024-11-24 08:29:32,486 main DEBUG Building Plugin[name=appender, class=org.apache.hadoop.hbase.logging.HBaseTestAppender]. 2024-11-24 08:29:32,488 main DEBUG HBaseTestAppender$Builder(target="SYSTEM_ERR", maxSize="1G", bufferedIo="null", bufferSize="null", immediateFlush="null", ignoreExceptions="null", PatternLayout(%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n), name="Console", Configuration(PropertiesConfig), Filter=null, ={}) 2024-11-24 08:29:32,490 main DEBUG Starting HBaseTestOutputStreamManager SYSTEM_ERR 2024-11-24 08:29:32,490 main DEBUG Building Plugin[name=appenders, class=org.apache.logging.log4j.core.config.AppendersPlugin]. 2024-11-24 08:29:32,490 main DEBUG createAppenders(={Console}) 2024-11-24 08:29:32,491 main DEBUG Configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@20b2475a initialized 2024-11-24 08:29:32,491 main DEBUG Starting configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@20b2475a 2024-11-24 08:29:32,491 main DEBUG Started configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@20b2475a OK. 2024-11-24 08:29:32,492 main DEBUG Shutting down OutputStreamManager SYSTEM_OUT.false.false-1 2024-11-24 08:29:32,492 main DEBUG OutputStream closed 2024-11-24 08:29:32,492 main DEBUG Shut down OutputStreamManager SYSTEM_OUT.false.false-1, all resources released: true 2024-11-24 08:29:32,493 main DEBUG Appender DefaultConsole-1 stopped with status true 2024-11-24 08:29:32,493 main DEBUG Stopped org.apache.logging.log4j.core.config.DefaultConfiguration@4310d43 OK 2024-11-24 08:29:32,559 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6 2024-11-24 08:29:32,561 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=StatusLogger 2024-11-24 08:29:32,562 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=ContextSelector 2024-11-24 08:29:32,563 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name= 2024-11-24 08:29:32,563 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.directory 2024-11-24 08:29:32,564 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSinkAdapter 2024-11-24 08:29:32,564 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.zookeeper 2024-11-24 08:29:32,564 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.logging.TestJul2Slf4j 2024-11-24 08:29:32,564 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSystemImpl 2024-11-24 08:29:32,565 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.util.MBeans 2024-11-24 08:29:32,565 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase 2024-11-24 08:29:32,565 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop 2024-11-24 08:29:32,566 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ipc.FailedServers 2024-11-24 08:29:32,566 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.regionserver.RSRpcServices 2024-11-24 08:29:32,566 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsConfig 2024-11-24 08:29:32,566 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hbase.thirdparty.io.netty.channel 2024-11-24 08:29:32,567 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ScheduledChore 2024-11-24 08:29:32,567 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Appenders,name=Console 2024-11-24 08:29:32,570 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-24 08:29:32,570 main DEBUG Reconfiguration complete for context[name=1dbd16a6] at URI jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-logging/target/hbase-logging-3.0.0-beta-2-SNAPSHOT-tests.jar!/log4j2.properties (org.apache.logging.log4j.core.LoggerContext@12f9af83) with optional ClassLoader: null 2024-11-24 08:29:32,570 main DEBUG Shutdown hook enabled. Registering a new one. 2024-11-24 08:29:32,571 main DEBUG LoggerContext[name=1dbd16a6, org.apache.logging.log4j.core.LoggerContext@12f9af83] started OK. 2024-11-24T08:29:32,799 DEBUG [main {}] hbase.HBaseTestingUtil(323): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ed430c83-cb94-7d1a-6ba6-ccf0f0e534d9 2024-11-24 08:29:32,802 main DEBUG AsyncLogger.ThreadNameStrategy=UNCACHED (user specified null, default is UNCACHED) 2024-11-24 08:29:32,802 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-24T08:29:32,811 INFO [main {}] hbase.HBaseClassTestRule(94): Test class org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay timeout: 13 mins 2024-11-24T08:29:32,817 INFO [main {}] hbase.HBaseClassTestRule(94): Test class org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplayCompressed timeout: 13 mins 2024-11-24T08:29:32,837 DEBUG [Time-limited test {}] channel.MultithreadEventLoopGroup(44): -Dio.netty.eventLoopThreads: 16 2024-11-24T08:29:32,883 DEBUG [Time-limited test {}] nio.NioEventLoop(110): -Dio.netty.noKeySetOptimization: false 2024-11-24T08:29:32,883 DEBUG [Time-limited test {}] nio.NioEventLoop(111): -Dio.netty.selectorAutoRebuildThreshold: 512 2024-11-24T08:29:32,897 INFO [Time-limited test {}] hbase.HBaseTestingUtil(805): Starting up minicluster with option: StartMiniClusterOption{numMasters=1, masterClass=null, numRegionServers=3, rsPorts=, rsClass=null, numDataNodes=3, dataNodeHosts=null, numZkServers=1, createRootDir=false, createWALDir=false} 2024-11-24T08:29:32,917 INFO [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ed430c83-cb94-7d1a-6ba6-ccf0f0e534d9/cluster_264a055b-3bad-7287-6b90-a8fe44ac1a0a, deleteOnExit=true 2024-11-24T08:29:32,917 INFO [Time-limited test {}] hbase.HBaseTestingUtil(818): STARTING DFS 2024-11-24T08:29:32,918 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ed430c83-cb94-7d1a-6ba6-ccf0f0e534d9/test.cache.data in system properties and HBase conf 2024-11-24T08:29:32,919 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ed430c83-cb94-7d1a-6ba6-ccf0f0e534d9/hadoop.tmp.dir in system properties and HBase conf 2024-11-24T08:29:32,919 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ed430c83-cb94-7d1a-6ba6-ccf0f0e534d9/hadoop.log.dir in system properties and HBase conf 2024-11-24T08:29:32,920 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ed430c83-cb94-7d1a-6ba6-ccf0f0e534d9/mapreduce.cluster.local.dir in system properties and HBase conf 2024-11-24T08:29:32,921 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ed430c83-cb94-7d1a-6ba6-ccf0f0e534d9/mapreduce.cluster.temp.dir in system properties and HBase conf 2024-11-24T08:29:32,921 INFO [Time-limited test {}] hbase.HBaseTestingUtil(738): read short circuit is OFF 2024-11-24T08:29:33,006 WARN [Time-limited test {}] util.NativeCodeLoader(60): Unable to load native-hadoop library for your platform... using builtin-java classes where applicable 2024-11-24T08:29:33,101 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-24T08:29:33,104 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ed430c83-cb94-7d1a-6ba6-ccf0f0e534d9/yarn.node-labels.fs-store.root-dir in system properties and HBase conf 2024-11-24T08:29:33,105 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ed430c83-cb94-7d1a-6ba6-ccf0f0e534d9/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf 2024-11-24T08:29:33,105 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ed430c83-cb94-7d1a-6ba6-ccf0f0e534d9/yarn.nodemanager.log-dirs in system properties and HBase conf 2024-11-24T08:29:33,106 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ed430c83-cb94-7d1a-6ba6-ccf0f0e534d9/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-11-24T08:29:33,106 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ed430c83-cb94-7d1a-6ba6-ccf0f0e534d9/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf 2024-11-24T08:29:33,107 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ed430c83-cb94-7d1a-6ba6-ccf0f0e534d9/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf 2024-11-24T08:29:33,107 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ed430c83-cb94-7d1a-6ba6-ccf0f0e534d9/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-11-24T08:29:33,107 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ed430c83-cb94-7d1a-6ba6-ccf0f0e534d9/dfs.journalnode.edits.dir in system properties and HBase conf 2024-11-24T08:29:33,108 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ed430c83-cb94-7d1a-6ba6-ccf0f0e534d9/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf 2024-11-24T08:29:33,108 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ed430c83-cb94-7d1a-6ba6-ccf0f0e534d9/nfs.dump.dir in system properties and HBase conf 2024-11-24T08:29:33,108 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ed430c83-cb94-7d1a-6ba6-ccf0f0e534d9/java.io.tmpdir in system properties and HBase conf 2024-11-24T08:29:33,109 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ed430c83-cb94-7d1a-6ba6-ccf0f0e534d9/dfs.journalnode.edits.dir in system properties and HBase conf 2024-11-24T08:29:33,109 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ed430c83-cb94-7d1a-6ba6-ccf0f0e534d9/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf 2024-11-24T08:29:33,109 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ed430c83-cb94-7d1a-6ba6-ccf0f0e534d9/fs.s3a.committer.staging.tmp.path in system properties and HBase conf 2024-11-24T08:29:33,927 WARN [Time-limited test {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-namenode.properties,hadoop-metrics2.properties 2024-11-24T08:29:34,007 INFO [Time-limited test {}] log.Log(170): Logging initialized @2279ms to org.eclipse.jetty.util.log.Slf4jLog 2024-11-24T08:29:34,093 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-24T08:29:34,162 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-24T08:29:34,182 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-24T08:29:34,182 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-24T08:29:34,184 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-11-24T08:29:34,207 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-24T08:29:34,210 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@6630be9c{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ed430c83-cb94-7d1a-6ba6-ccf0f0e534d9/hadoop.log.dir/,AVAILABLE} 2024-11-24T08:29:34,211 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@70596a48{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-24T08:29:34,438 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@1843526c{hdfs,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ed430c83-cb94-7d1a-6ba6-ccf0f0e534d9/java.io.tmpdir/jetty-localhost-40551-hadoop-hdfs-3_4_1-tests_jar-_-any-914073725902792335/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-11-24T08:29:34,449 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@776c4d96{HTTP/1.1, (http/1.1)}{localhost:40551} 2024-11-24T08:29:34,450 INFO [Time-limited test {}] server.Server(415): Started @2722ms 2024-11-24T08:29:34,853 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-24T08:29:34,859 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-24T08:29:34,860 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-24T08:29:34,860 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-24T08:29:34,860 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2024-11-24T08:29:34,862 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@4b895d95{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ed430c83-cb94-7d1a-6ba6-ccf0f0e534d9/hadoop.log.dir/,AVAILABLE} 2024-11-24T08:29:34,863 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@3d33512e{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-24T08:29:34,982 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@4dc44ec6{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ed430c83-cb94-7d1a-6ba6-ccf0f0e534d9/java.io.tmpdir/jetty-localhost-34293-hadoop-hdfs-3_4_1-tests_jar-_-any-2808301577174674947/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-24T08:29:34,983 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@17f6e478{HTTP/1.1, (http/1.1)}{localhost:34293} 2024-11-24T08:29:34,983 INFO [Time-limited test {}] server.Server(415): Started @3256ms 2024-11-24T08:29:35,036 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-11-24T08:29:35,164 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-24T08:29:35,171 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-24T08:29:35,174 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-24T08:29:35,175 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-24T08:29:35,175 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-11-24T08:29:35,176 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@e971547{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ed430c83-cb94-7d1a-6ba6-ccf0f0e534d9/hadoop.log.dir/,AVAILABLE} 2024-11-24T08:29:35,177 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@3264f7d{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-24T08:29:35,322 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@28e1ba78{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ed430c83-cb94-7d1a-6ba6-ccf0f0e534d9/java.io.tmpdir/jetty-localhost-36069-hadoop-hdfs-3_4_1-tests_jar-_-any-9870798925293469389/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-24T08:29:35,323 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@702c0733{HTTP/1.1, (http/1.1)}{localhost:36069} 2024-11-24T08:29:35,324 INFO [Time-limited test {}] server.Server(415): Started @3597ms 2024-11-24T08:29:35,326 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-11-24T08:29:35,375 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-24T08:29:35,379 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-24T08:29:35,381 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-24T08:29:35,381 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-24T08:29:35,381 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-11-24T08:29:35,382 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@1bf0fdb4{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ed430c83-cb94-7d1a-6ba6-ccf0f0e534d9/hadoop.log.dir/,AVAILABLE} 2024-11-24T08:29:35,382 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@6853f5bc{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-24T08:29:35,493 WARN [Thread-105 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ed430c83-cb94-7d1a-6ba6-ccf0f0e534d9/cluster_264a055b-3bad-7287-6b90-a8fe44ac1a0a/data/data3/current/BP-1907186785-172.17.0.2-1732436973683/current, will proceed with Du for space computation calculation, 2024-11-24T08:29:35,493 WARN [Thread-108 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ed430c83-cb94-7d1a-6ba6-ccf0f0e534d9/cluster_264a055b-3bad-7287-6b90-a8fe44ac1a0a/data/data2/current/BP-1907186785-172.17.0.2-1732436973683/current, will proceed with Du for space computation calculation, 2024-11-24T08:29:35,493 WARN [Thread-107 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ed430c83-cb94-7d1a-6ba6-ccf0f0e534d9/cluster_264a055b-3bad-7287-6b90-a8fe44ac1a0a/data/data4/current/BP-1907186785-172.17.0.2-1732436973683/current, will proceed with Du for space computation calculation, 2024-11-24T08:29:35,493 WARN [Thread-106 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ed430c83-cb94-7d1a-6ba6-ccf0f0e534d9/cluster_264a055b-3bad-7287-6b90-a8fe44ac1a0a/data/data1/current/BP-1907186785-172.17.0.2-1732436973683/current, will proceed with Du for space computation calculation, 2024-11-24T08:29:35,542 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@4290616c{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ed430c83-cb94-7d1a-6ba6-ccf0f0e534d9/java.io.tmpdir/jetty-localhost-41867-hadoop-hdfs-3_4_1-tests_jar-_-any-13296077918474400460/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-24T08:29:35,543 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@5e25d2b6{HTTP/1.1, (http/1.1)}{localhost:41867} 2024-11-24T08:29:35,544 INFO [Time-limited test {}] server.Server(415): Started @3817ms 2024-11-24T08:29:35,544 WARN [Thread-82 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-11-24T08:29:35,545 WARN [Thread-58 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-11-24T08:29:35,547 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-11-24T08:29:35,613 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xd6b2befe19447796 with lease ID 0xf84f631ac0856d1: Processing first storage report for DS-5eeb9666-6dd5-4a48-9fb1-3f9aff1acdca from datanode DatanodeRegistration(127.0.0.1:40533, datanodeUuid=e762698c-7e76-4e21-b6cc-6263e64f5953, infoPort=42673, infoSecurePort=0, ipcPort=40413, storageInfo=lv=-57;cid=testClusterID;nsid=444601671;c=1732436973683) 2024-11-24T08:29:35,614 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xd6b2befe19447796 with lease ID 0xf84f631ac0856d1: from storage DS-5eeb9666-6dd5-4a48-9fb1-3f9aff1acdca node DatanodeRegistration(127.0.0.1:40533, datanodeUuid=e762698c-7e76-4e21-b6cc-6263e64f5953, infoPort=42673, infoSecurePort=0, ipcPort=40413, storageInfo=lv=-57;cid=testClusterID;nsid=444601671;c=1732436973683), blocks: 0, hasStaleStorage: true, processing time: 2 msecs, invalidatedBlocks: 0 2024-11-24T08:29:35,615 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xb83ec033fb3bd3e1 with lease ID 0xf84f631ac0856d0: Processing first storage report for DS-6b4d1656-f49b-4a16-be49-2eb682c16e11 from datanode DatanodeRegistration(127.0.0.1:34459, datanodeUuid=3237fd78-a6e4-4c92-99cc-6bca7ba6e9c2, infoPort=44131, infoSecurePort=0, ipcPort=34731, storageInfo=lv=-57;cid=testClusterID;nsid=444601671;c=1732436973683) 2024-11-24T08:29:35,615 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xb83ec033fb3bd3e1 with lease ID 0xf84f631ac0856d0: from storage DS-6b4d1656-f49b-4a16-be49-2eb682c16e11 node DatanodeRegistration(127.0.0.1:34459, datanodeUuid=3237fd78-a6e4-4c92-99cc-6bca7ba6e9c2, infoPort=44131, infoSecurePort=0, ipcPort=34731, storageInfo=lv=-57;cid=testClusterID;nsid=444601671;c=1732436973683), blocks: 0, hasStaleStorage: true, processing time: 1 msecs, invalidatedBlocks: 0 2024-11-24T08:29:35,615 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xd6b2befe19447796 with lease ID 0xf84f631ac0856d1: Processing first storage report for DS-34fa01d9-5956-47f0-ae3d-1dbcef269609 from datanode DatanodeRegistration(127.0.0.1:40533, datanodeUuid=e762698c-7e76-4e21-b6cc-6263e64f5953, infoPort=42673, infoSecurePort=0, ipcPort=40413, storageInfo=lv=-57;cid=testClusterID;nsid=444601671;c=1732436973683) 2024-11-24T08:29:35,615 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xd6b2befe19447796 with lease ID 0xf84f631ac0856d1: from storage DS-34fa01d9-5956-47f0-ae3d-1dbcef269609 node DatanodeRegistration(127.0.0.1:40533, datanodeUuid=e762698c-7e76-4e21-b6cc-6263e64f5953, infoPort=42673, infoSecurePort=0, ipcPort=40413, storageInfo=lv=-57;cid=testClusterID;nsid=444601671;c=1732436973683), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-24T08:29:35,616 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xb83ec033fb3bd3e1 with lease ID 0xf84f631ac0856d0: Processing first storage report for DS-7348a281-7681-4235-aa7e-69fa8049db1d from datanode DatanodeRegistration(127.0.0.1:34459, datanodeUuid=3237fd78-a6e4-4c92-99cc-6bca7ba6e9c2, infoPort=44131, infoSecurePort=0, ipcPort=34731, storageInfo=lv=-57;cid=testClusterID;nsid=444601671;c=1732436973683) 2024-11-24T08:29:35,616 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xb83ec033fb3bd3e1 with lease ID 0xf84f631ac0856d0: from storage DS-7348a281-7681-4235-aa7e-69fa8049db1d node DatanodeRegistration(127.0.0.1:34459, datanodeUuid=3237fd78-a6e4-4c92-99cc-6bca7ba6e9c2, infoPort=44131, infoSecurePort=0, ipcPort=34731, storageInfo=lv=-57;cid=testClusterID;nsid=444601671;c=1732436973683), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-24T08:29:35,691 WARN [Thread-140 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ed430c83-cb94-7d1a-6ba6-ccf0f0e534d9/cluster_264a055b-3bad-7287-6b90-a8fe44ac1a0a/data/data6/current/BP-1907186785-172.17.0.2-1732436973683/current, will proceed with Du for space computation calculation, 2024-11-24T08:29:35,691 WARN [Thread-139 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ed430c83-cb94-7d1a-6ba6-ccf0f0e534d9/cluster_264a055b-3bad-7287-6b90-a8fe44ac1a0a/data/data5/current/BP-1907186785-172.17.0.2-1732436973683/current, will proceed with Du for space computation calculation, 2024-11-24T08:29:35,723 WARN [Thread-129 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-11-24T08:29:35,731 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x37730411108142f7 with lease ID 0xf84f631ac0856d2: Processing first storage report for DS-8d97c3e2-1525-418c-a970-00a7157a0d31 from datanode DatanodeRegistration(127.0.0.1:33555, datanodeUuid=8a1b4064-c118-49d3-bb11-efcad72fe18b, infoPort=39357, infoSecurePort=0, ipcPort=41943, storageInfo=lv=-57;cid=testClusterID;nsid=444601671;c=1732436973683) 2024-11-24T08:29:35,732 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x37730411108142f7 with lease ID 0xf84f631ac0856d2: from storage DS-8d97c3e2-1525-418c-a970-00a7157a0d31 node DatanodeRegistration(127.0.0.1:33555, datanodeUuid=8a1b4064-c118-49d3-bb11-efcad72fe18b, infoPort=39357, infoSecurePort=0, ipcPort=41943, storageInfo=lv=-57;cid=testClusterID;nsid=444601671;c=1732436973683), blocks: 0, hasStaleStorage: true, processing time: 1 msecs, invalidatedBlocks: 0 2024-11-24T08:29:35,732 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x37730411108142f7 with lease ID 0xf84f631ac0856d2: Processing first storage report for DS-d85f8240-519a-41a7-a228-95971e1e81ce from datanode DatanodeRegistration(127.0.0.1:33555, datanodeUuid=8a1b4064-c118-49d3-bb11-efcad72fe18b, infoPort=39357, infoSecurePort=0, ipcPort=41943, storageInfo=lv=-57;cid=testClusterID;nsid=444601671;c=1732436973683) 2024-11-24T08:29:35,732 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x37730411108142f7 with lease ID 0xf84f631ac0856d2: from storage DS-d85f8240-519a-41a7-a228-95971e1e81ce node DatanodeRegistration(127.0.0.1:33555, datanodeUuid=8a1b4064-c118-49d3-bb11-efcad72fe18b, infoPort=39357, infoSecurePort=0, ipcPort=41943, storageInfo=lv=-57;cid=testClusterID;nsid=444601671;c=1732436973683), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-24T08:29:35,959 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(631): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ed430c83-cb94-7d1a-6ba6-ccf0f0e534d9 2024-11-24T08:29:36,055 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ed430c83-cb94-7d1a-6ba6-ccf0f0e534d9/cluster_264a055b-3bad-7287-6b90-a8fe44ac1a0a/zookeeper_0, clientPort=59068, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ed430c83-cb94-7d1a-6ba6-ccf0f0e534d9/cluster_264a055b-3bad-7287-6b90-a8fe44ac1a0a/zookeeper_0/version-2, dataDirSize=457 dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ed430c83-cb94-7d1a-6ba6-ccf0f0e534d9/cluster_264a055b-3bad-7287-6b90-a8fe44ac1a0a/zookeeper_0/version-2, dataLogSize=457 tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0 2024-11-24T08:29:36,067 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=59068 2024-11-24T08:29:36,081 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-24T08:29:36,084 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-24T08:29:36,306 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33555 is added to blk_1073741825_1001 (size=7) 2024-11-24T08:29:36,306 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40533 is added to blk_1073741825_1001 (size=7) 2024-11-24T08:29:36,307 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34459 is added to blk_1073741825_1001 (size=7) 2024-11-24T08:29:36,715 INFO [Time-limited test {}] util.FSUtils(489): Created version file at hdfs://localhost:46165/user/jenkins/test-data/7507fec3-e300-7585-3a43-9c9d3ed5b09e with version=8 2024-11-24T08:29:36,715 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1139): Setting hbase.fs.tmp.dir to hdfs://localhost:46165/user/jenkins/test-data/7507fec3-e300-7585-3a43-9c9d3ed5b09e/hbase-staging 2024-11-24T08:29:37,040 INFO [Time-limited test {}] client.ConnectionUtils(128): master/30c28c82771d:0 server-side Connection retries=45 2024-11-24T08:29:37,052 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-24T08:29:37,052 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-24T08:29:37,058 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-24T08:29:37,059 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-24T08:29:37,059 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-24T08:29:37,199 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService 2024-11-24T08:29:37,261 INFO [Time-limited test {}] metrics.MetricRegistriesLoader(60): Loaded MetricRegistries class org.apache.hadoop.hbase.metrics.impl.MetricRegistriesImpl 2024-11-24T08:29:37,270 DEBUG [Time-limited test {}] util.ClassSize(228): Using Unsafe to estimate memory layout 2024-11-24T08:29:37,274 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-24T08:29:37,301 DEBUG [Time-limited test {}] channel.DefaultChannelId(84): -Dio.netty.processId: 22860 (auto-detected) 2024-11-24T08:29:37,302 DEBUG [Time-limited test {}] channel.DefaultChannelId(106): -Dio.netty.machineId: 02:42:ac:ff:fe:11:00:02 (auto-detected) 2024-11-24T08:29:37,322 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:37719 2024-11-24T08:29:37,344 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:37719 connecting to ZooKeeper ensemble=127.0.0.1:59068 2024-11-24T08:29:37,380 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:377190x0, quorum=127.0.0.1:59068, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-24T08:29:37,383 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:37719-0x10149184c260000 connected 2024-11-24T08:29:37,413 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-24T08:29:37,418 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-24T08:29:37,432 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:37719-0x10149184c260000, quorum=127.0.0.1:59068, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-24T08:29:37,437 INFO [Time-limited test {}] master.HMaster(525): hbase.rootdir=hdfs://localhost:46165/user/jenkins/test-data/7507fec3-e300-7585-3a43-9c9d3ed5b09e, hbase.cluster.distributed=false 2024-11-24T08:29:37,460 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:37719-0x10149184c260000, quorum=127.0.0.1:59068, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-24T08:29:37,465 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=37719 2024-11-24T08:29:37,466 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=37719 2024-11-24T08:29:37,466 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=37719 2024-11-24T08:29:37,467 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=37719 2024-11-24T08:29:37,467 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=37719 2024-11-24T08:29:37,598 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/30c28c82771d:0 server-side Connection retries=45 2024-11-24T08:29:37,600 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-24T08:29:37,600 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-24T08:29:37,600 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-24T08:29:37,600 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-24T08:29:37,600 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-24T08:29:37,603 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-24T08:29:37,605 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-24T08:29:37,606 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:40117 2024-11-24T08:29:37,609 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:40117 connecting to ZooKeeper ensemble=127.0.0.1:59068 2024-11-24T08:29:37,610 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-24T08:29:37,613 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-24T08:29:37,620 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:401170x0, quorum=127.0.0.1:59068, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-24T08:29:37,621 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:40117-0x10149184c260001 connected 2024-11-24T08:29:37,621 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:40117-0x10149184c260001, quorum=127.0.0.1:59068, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-24T08:29:37,627 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-24T08:29:37,639 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-24T08:29:37,642 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:40117-0x10149184c260001, quorum=127.0.0.1:59068, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-24T08:29:37,648 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:40117-0x10149184c260001, quorum=127.0.0.1:59068, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-24T08:29:37,650 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=40117 2024-11-24T08:29:37,650 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=40117 2024-11-24T08:29:37,651 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=40117 2024-11-24T08:29:37,652 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=40117 2024-11-24T08:29:37,652 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=40117 2024-11-24T08:29:37,670 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/30c28c82771d:0 server-side Connection retries=45 2024-11-24T08:29:37,670 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-24T08:29:37,670 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-24T08:29:37,671 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-24T08:29:37,671 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-24T08:29:37,671 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-24T08:29:37,671 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-24T08:29:37,672 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-24T08:29:37,675 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:40241 2024-11-24T08:29:37,677 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:40241 connecting to ZooKeeper ensemble=127.0.0.1:59068 2024-11-24T08:29:37,678 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-24T08:29:37,680 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-24T08:29:37,686 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:402410x0, quorum=127.0.0.1:59068, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-24T08:29:37,687 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:402410x0, quorum=127.0.0.1:59068, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-24T08:29:37,687 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-24T08:29:37,689 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:40241-0x10149184c260002 connected 2024-11-24T08:29:37,691 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-24T08:29:37,692 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:40241-0x10149184c260002, quorum=127.0.0.1:59068, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-24T08:29:37,695 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:40241-0x10149184c260002, quorum=127.0.0.1:59068, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-24T08:29:37,696 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=40241 2024-11-24T08:29:37,697 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=40241 2024-11-24T08:29:37,697 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=40241 2024-11-24T08:29:37,698 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=40241 2024-11-24T08:29:37,698 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=40241 2024-11-24T08:29:37,717 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/30c28c82771d:0 server-side Connection retries=45 2024-11-24T08:29:37,717 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-24T08:29:37,717 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-24T08:29:37,717 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-24T08:29:37,717 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-24T08:29:37,717 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-24T08:29:37,718 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-24T08:29:37,718 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-24T08:29:37,719 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:38023 2024-11-24T08:29:37,721 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:38023 connecting to ZooKeeper ensemble=127.0.0.1:59068 2024-11-24T08:29:37,722 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-24T08:29:37,725 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-24T08:29:37,730 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:380230x0, quorum=127.0.0.1:59068, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-24T08:29:37,731 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:380230x0, quorum=127.0.0.1:59068, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-24T08:29:37,732 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-24T08:29:37,733 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:38023-0x10149184c260003 connected 2024-11-24T08:29:37,733 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-24T08:29:37,734 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:38023-0x10149184c260003, quorum=127.0.0.1:59068, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-24T08:29:37,736 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:38023-0x10149184c260003, quorum=127.0.0.1:59068, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-24T08:29:37,739 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=38023 2024-11-24T08:29:37,740 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=38023 2024-11-24T08:29:37,741 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=38023 2024-11-24T08:29:37,744 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=38023 2024-11-24T08:29:37,745 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=38023 2024-11-24T08:29:37,768 DEBUG [M:0;30c28c82771d:37719 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;30c28c82771d:37719 2024-11-24T08:29:37,769 INFO [master/30c28c82771d:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/30c28c82771d,37719,1732436976835 2024-11-24T08:29:37,777 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37719-0x10149184c260000, quorum=127.0.0.1:59068, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-24T08:29:37,777 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40241-0x10149184c260002, quorum=127.0.0.1:59068, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-24T08:29:37,778 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40117-0x10149184c260001, quorum=127.0.0.1:59068, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-24T08:29:37,778 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38023-0x10149184c260003, quorum=127.0.0.1:59068, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-24T08:29:37,780 DEBUG [master/30c28c82771d:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:37719-0x10149184c260000, quorum=127.0.0.1:59068, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/30c28c82771d,37719,1732436976835 2024-11-24T08:29:37,802 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40241-0x10149184c260002, quorum=127.0.0.1:59068, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-24T08:29:37,802 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38023-0x10149184c260003, quorum=127.0.0.1:59068, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-24T08:29:37,803 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37719-0x10149184c260000, quorum=127.0.0.1:59068, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-24T08:29:37,803 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38023-0x10149184c260003, quorum=127.0.0.1:59068, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-24T08:29:37,803 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40117-0x10149184c260001, quorum=127.0.0.1:59068, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-24T08:29:37,803 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40241-0x10149184c260002, quorum=127.0.0.1:59068, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-24T08:29:37,803 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40117-0x10149184c260001, quorum=127.0.0.1:59068, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-24T08:29:37,804 DEBUG [master/30c28c82771d:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:37719-0x10149184c260000, quorum=127.0.0.1:59068, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-11-24T08:29:37,805 INFO [master/30c28c82771d:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/30c28c82771d,37719,1732436976835 from backup master directory 2024-11-24T08:29:37,808 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37719-0x10149184c260000, quorum=127.0.0.1:59068, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/30c28c82771d,37719,1732436976835 2024-11-24T08:29:37,808 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40241-0x10149184c260002, quorum=127.0.0.1:59068, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-24T08:29:37,808 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38023-0x10149184c260003, quorum=127.0.0.1:59068, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-24T08:29:37,808 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37719-0x10149184c260000, quorum=127.0.0.1:59068, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-24T08:29:37,808 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40117-0x10149184c260001, quorum=127.0.0.1:59068, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-24T08:29:37,809 WARN [master/30c28c82771d:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-24T08:29:37,810 INFO [master/30c28c82771d:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=30c28c82771d,37719,1732436976835 2024-11-24T08:29:37,812 INFO [master/30c28c82771d:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating data MemStoreChunkPool with chunk size 2 MB, max count 396, initial count 0 2024-11-24T08:29:37,813 INFO [master/30c28c82771d:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating index MemStoreChunkPool with chunk size 204.80 KB, max count 440, initial count 0 2024-11-24T08:29:37,870 DEBUG [master/30c28c82771d:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [hdfs://localhost:46165/user/jenkins/test-data/7507fec3-e300-7585-3a43-9c9d3ed5b09e/hbase.id] with ID: 8962100e-ff2b-4a0c-9fd7-8de6412af7e9 2024-11-24T08:29:37,870 DEBUG [master/30c28c82771d:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: hdfs://localhost:46165/user/jenkins/test-data/7507fec3-e300-7585-3a43-9c9d3ed5b09e/.tmp/hbase.id 2024-11-24T08:29:37,884 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34459 is added to blk_1073741826_1002 (size=42) 2024-11-24T08:29:37,885 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40533 is added to blk_1073741826_1002 (size=42) 2024-11-24T08:29:37,885 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33555 is added to blk_1073741826_1002 (size=42) 2024-11-24T08:29:37,886 DEBUG [master/30c28c82771d:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [hdfs://localhost:46165/user/jenkins/test-data/7507fec3-e300-7585-3a43-9c9d3ed5b09e/.tmp/hbase.id]:[hdfs://localhost:46165/user/jenkins/test-data/7507fec3-e300-7585-3a43-9c9d3ed5b09e/hbase.id] 2024-11-24T08:29:37,932 INFO [master/30c28c82771d:0:becomeActiveMaster {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-24T08:29:37,937 INFO [master/30c28c82771d:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem. 2024-11-24T08:29:37,957 INFO [master/30c28c82771d:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 18ms. 2024-11-24T08:29:37,961 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38023-0x10149184c260003, quorum=127.0.0.1:59068, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-24T08:29:37,961 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40117-0x10149184c260001, quorum=127.0.0.1:59068, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-24T08:29:37,961 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37719-0x10149184c260000, quorum=127.0.0.1:59068, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-24T08:29:37,961 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40241-0x10149184c260002, quorum=127.0.0.1:59068, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-24T08:29:37,980 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40533 is added to blk_1073741827_1003 (size=196) 2024-11-24T08:29:37,980 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34459 is added to blk_1073741827_1003 (size=196) 2024-11-24T08:29:37,981 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33555 is added to blk_1073741827_1003 (size=196) 2024-11-24T08:29:37,997 INFO [master/30c28c82771d:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-24T08:29:37,999 INFO [master/30c28c82771d:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2024-11-24T08:29:38,005 INFO [master/30c28c82771d:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-24T08:29:38,040 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40533 is added to blk_1073741828_1004 (size=1189) 2024-11-24T08:29:38,041 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34459 is added to blk_1073741828_1004 (size=1189) 2024-11-24T08:29:38,041 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33555 is added to blk_1073741828_1004 (size=1189) 2024-11-24T08:29:38,061 INFO [master/30c28c82771d:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir hdfs://localhost:46165/user/jenkins/test-data/7507fec3-e300-7585-3a43-9c9d3ed5b09e/MasterData/data/master/store 2024-11-24T08:29:38,083 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33555 is added to blk_1073741829_1005 (size=34) 2024-11-24T08:29:38,083 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34459 is added to blk_1073741829_1005 (size=34) 2024-11-24T08:29:38,085 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40533 is added to blk_1073741829_1005 (size=34) 2024-11-24T08:29:38,092 INFO [master/30c28c82771d:0:becomeActiveMaster {}] throttle.StoreHotnessProtector(112): StoreHotnessProtector is disabled. Set hbase.region.store.parallel.put.limit > 0 to enable, which may help mitigate load under heavy write pressure. 2024-11-24T08:29:38,096 DEBUG [master/30c28c82771d:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T08:29:38,098 DEBUG [master/30c28c82771d:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-24T08:29:38,098 INFO [master/30c28c82771d:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-24T08:29:38,098 DEBUG [master/30c28c82771d:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-24T08:29:38,100 DEBUG [master/30c28c82771d:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-24T08:29:38,100 DEBUG [master/30c28c82771d:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-24T08:29:38,101 INFO [master/30c28c82771d:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-24T08:29:38,102 DEBUG [master/30c28c82771d:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1732436978097Disabling compacts and flushes for region at 1732436978097Disabling writes for close at 1732436978100 (+3 ms)Writing region close event to WAL at 1732436978101 (+1 ms)Closed at 1732436978101 2024-11-24T08:29:38,104 WARN [master/30c28c82771d:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: hdfs://localhost:46165/user/jenkins/test-data/7507fec3-e300-7585-3a43-9c9d3ed5b09e/MasterData/data/master/store/.initializing 2024-11-24T08:29:38,105 DEBUG [master/30c28c82771d:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=hdfs://localhost:46165/user/jenkins/test-data/7507fec3-e300-7585-3a43-9c9d3ed5b09e/MasterData/WALs/30c28c82771d,37719,1732436976835 2024-11-24T08:29:38,114 INFO [master/30c28c82771d:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-24T08:29:38,134 INFO [master/30c28c82771d:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=30c28c82771d%2C37719%2C1732436976835, suffix=, logDir=hdfs://localhost:46165/user/jenkins/test-data/7507fec3-e300-7585-3a43-9c9d3ed5b09e/MasterData/WALs/30c28c82771d,37719,1732436976835, archiveDir=hdfs://localhost:46165/user/jenkins/test-data/7507fec3-e300-7585-3a43-9c9d3ed5b09e/MasterData/oldWALs, maxLogs=10 2024-11-24T08:29:38,176 DEBUG [master/30c28c82771d:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/7507fec3-e300-7585-3a43-9c9d3ed5b09e/MasterData/WALs/30c28c82771d,37719,1732436976835/30c28c82771d%2C37719%2C1732436976835.1732436978141, exclude list is [], retry=0 2024-11-24T08:29:38,194 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(244): No decryptEncryptedDataEncryptionKey method in DFSClient, should be hadoop version with HDFS-12396 java.lang.NoSuchMethodException: org.apache.hadoop.hdfs.DFSClient.decryptEncryptedDataEncryptionKey(org.apache.hadoop.fs.FileEncryptionInfo) at java.lang.Class.getDeclaredMethod(Class.java:2675) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelperWithoutHDFS12396(FanOutOneBlockAsyncDFSOutputSaslHelper.java:183) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelper(FanOutOneBlockAsyncDFSOutputSaslHelper.java:242) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.(FanOutOneBlockAsyncDFSOutputSaslHelper.java:253) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.initialize(FanOutOneBlockAsyncDFSOutputHelper.java:413) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper$5.operationComplete(FanOutOneBlockAsyncDFSOutputHelper.java:472) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper$5.operationComplete(FanOutOneBlockAsyncDFSOutputHelper.java:467) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.NettyFutureUtils.lambda$addListener$0(NettyFutureUtils.java:56) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setSuccess0(DefaultPromise.java:625) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.trySuccess(DefaultPromise.java:105) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPromise.trySuccess(DefaultChannelPromise.java:84) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.fulfillConnectPromise(AbstractEpollChannel.java:658) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:696) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:491) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T08:29:38,199 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33555,DS-8d97c3e2-1525-418c-a970-00a7157a0d31,DISK] 2024-11-24T08:29:38,199 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:34459,DS-6b4d1656-f49b-4a16-be49-2eb682c16e11,DISK] 2024-11-24T08:29:38,199 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40533,DS-5eeb9666-6dd5-4a48-9fb1-3f9aff1acdca,DISK] 2024-11-24T08:29:38,202 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.ProtobufDecoder(117): Hadoop 3.3 and above shades protobuf. 2024-11-24T08:29:38,242 INFO [master/30c28c82771d:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/7507fec3-e300-7585-3a43-9c9d3ed5b09e/MasterData/WALs/30c28c82771d,37719,1732436976835/30c28c82771d%2C37719%2C1732436976835.1732436978141 2024-11-24T08:29:38,244 DEBUG [master/30c28c82771d:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:39357:39357),(127.0.0.1/127.0.0.1:42673:42673),(127.0.0.1/127.0.0.1:44131:44131)] 2024-11-24T08:29:38,244 DEBUG [master/30c28c82771d:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2024-11-24T08:29:38,245 DEBUG [master/30c28c82771d:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T08:29:38,249 DEBUG [master/30c28c82771d:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2024-11-24T08:29:38,250 DEBUG [master/30c28c82771d:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2024-11-24T08:29:38,287 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2024-11-24T08:29:38,315 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2024-11-24T08:29:38,318 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:29:38,321 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-24T08:29:38,322 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2024-11-24T08:29:38,325 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2024-11-24T08:29:38,325 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:29:38,326 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T08:29:38,326 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2024-11-24T08:29:38,329 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2024-11-24T08:29:38,329 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:29:38,330 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T08:29:38,330 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2024-11-24T08:29:38,333 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2024-11-24T08:29:38,333 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:29:38,334 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T08:29:38,334 DEBUG [master/30c28c82771d:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682 2024-11-24T08:29:38,337 DEBUG [master/30c28c82771d:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46165/user/jenkins/test-data/7507fec3-e300-7585-3a43-9c9d3ed5b09e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-24T08:29:38,338 DEBUG [master/30c28c82771d:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46165/user/jenkins/test-data/7507fec3-e300-7585-3a43-9c9d3ed5b09e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-24T08:29:38,343 DEBUG [master/30c28c82771d:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682 2024-11-24T08:29:38,344 DEBUG [master/30c28c82771d:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682 2024-11-24T08:29:38,347 DEBUG [master/30c28c82771d:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-24T08:29:38,350 DEBUG [master/30c28c82771d:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2024-11-24T08:29:38,355 DEBUG [master/30c28c82771d:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:46165/user/jenkins/test-data/7507fec3-e300-7585-3a43-9c9d3ed5b09e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-24T08:29:38,356 INFO [master/30c28c82771d:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=68419643, jitterRate=0.019532129168510437}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-24T08:29:38,362 DEBUG [master/30c28c82771d:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682: Writing region info on filesystem at 1732436978263Initializing all the Stores at 1732436978265 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732436978266 (+1 ms)Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732436978266Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732436978266Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732436978267 (+1 ms)Cleaning up temporary data from old regions at 1732436978344 (+77 ms)Region opened successfully at 1732436978361 (+17 ms) 2024-11-24T08:29:38,363 INFO [master/30c28c82771d:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2024-11-24T08:29:38,407 DEBUG [master/30c28c82771d:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1234aa40, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=30c28c82771d/172.17.0.2:0 2024-11-24T08:29:38,440 INFO [master/30c28c82771d:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating... 2024-11-24T08:29:38,452 INFO [master/30c28c82771d:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5 2024-11-24T08:29:38,452 INFO [master/30c28c82771d:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50 2024-11-24T08:29:38,455 INFO [master/30c28c82771d:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-24T08:29:38,457 INFO [master/30c28c82771d:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 1 msec 2024-11-24T08:29:38,462 INFO [master/30c28c82771d:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 4 msec 2024-11-24T08:29:38,462 INFO [master/30c28c82771d:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2024-11-24T08:29:38,493 INFO [master/30c28c82771d:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2024-11-24T08:29:38,504 DEBUG [master/30c28c82771d:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:37719-0x10149184c260000, quorum=127.0.0.1:59068, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error) 2024-11-24T08:29:38,507 DEBUG [master/30c28c82771d:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false 2024-11-24T08:29:38,510 INFO [master/30c28c82771d:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2024-11-24T08:29:38,512 DEBUG [master/30c28c82771d:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:37719-0x10149184c260000, quorum=127.0.0.1:59068, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error) 2024-11-24T08:29:38,513 DEBUG [master/30c28c82771d:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false 2024-11-24T08:29:38,515 INFO [master/30c28c82771d:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2024-11-24T08:29:38,519 DEBUG [master/30c28c82771d:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:37719-0x10149184c260000, quorum=127.0.0.1:59068, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error) 2024-11-24T08:29:38,520 DEBUG [master/30c28c82771d:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false 2024-11-24T08:29:38,521 DEBUG [master/30c28c82771d:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:37719-0x10149184c260000, quorum=127.0.0.1:59068, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error) 2024-11-24T08:29:38,523 DEBUG [master/30c28c82771d:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false 2024-11-24T08:29:38,542 DEBUG [master/30c28c82771d:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:37719-0x10149184c260000, quorum=127.0.0.1:59068, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error) 2024-11-24T08:29:38,544 DEBUG [master/30c28c82771d:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false 2024-11-24T08:29:38,548 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37719-0x10149184c260000, quorum=127.0.0.1:59068, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-24T08:29:38,548 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40241-0x10149184c260002, quorum=127.0.0.1:59068, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-24T08:29:38,548 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40117-0x10149184c260001, quorum=127.0.0.1:59068, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-24T08:29:38,548 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38023-0x10149184c260003, quorum=127.0.0.1:59068, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-24T08:29:38,549 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40117-0x10149184c260001, quorum=127.0.0.1:59068, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-24T08:29:38,549 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38023-0x10149184c260003, quorum=127.0.0.1:59068, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-24T08:29:38,549 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40241-0x10149184c260002, quorum=127.0.0.1:59068, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-24T08:29:38,549 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37719-0x10149184c260000, quorum=127.0.0.1:59068, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-24T08:29:38,552 INFO [master/30c28c82771d:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=30c28c82771d,37719,1732436976835, sessionid=0x10149184c260000, setting cluster-up flag (Was=false) 2024-11-24T08:29:38,567 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37719-0x10149184c260000, quorum=127.0.0.1:59068, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-24T08:29:38,567 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40117-0x10149184c260001, quorum=127.0.0.1:59068, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-24T08:29:38,567 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38023-0x10149184c260003, quorum=127.0.0.1:59068, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-24T08:29:38,567 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40241-0x10149184c260002, quorum=127.0.0.1:59068, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-24T08:29:38,575 DEBUG [master/30c28c82771d:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort 2024-11-24T08:29:38,576 DEBUG [master/30c28c82771d:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=30c28c82771d,37719,1732436976835 2024-11-24T08:29:38,583 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37719-0x10149184c260000, quorum=127.0.0.1:59068, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-24T08:29:38,583 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40241-0x10149184c260002, quorum=127.0.0.1:59068, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-24T08:29:38,583 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40117-0x10149184c260001, quorum=127.0.0.1:59068, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-24T08:29:38,583 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38023-0x10149184c260003, quorum=127.0.0.1:59068, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-24T08:29:38,589 DEBUG [master/30c28c82771d:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort 2024-11-24T08:29:38,590 DEBUG [master/30c28c82771d:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=30c28c82771d,37719,1732436976835 2024-11-24T08:29:38,597 INFO [master/30c28c82771d:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at hdfs://localhost:46165/user/jenkins/test-data/7507fec3-e300-7585-3a43-9c9d3ed5b09e/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again 2024-11-24T08:29:38,651 INFO [RS:0;30c28c82771d:40117 {}] regionserver.HRegionServer(746): ClusterId : 8962100e-ff2b-4a0c-9fd7-8de6412af7e9 2024-11-24T08:29:38,651 INFO [RS:2;30c28c82771d:38023 {}] regionserver.HRegionServer(746): ClusterId : 8962100e-ff2b-4a0c-9fd7-8de6412af7e9 2024-11-24T08:29:38,651 INFO [RS:1;30c28c82771d:40241 {}] regionserver.HRegionServer(746): ClusterId : 8962100e-ff2b-4a0c-9fd7-8de6412af7e9 2024-11-24T08:29:38,654 DEBUG [RS:2;30c28c82771d:38023 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-24T08:29:38,654 DEBUG [RS:0;30c28c82771d:40117 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-24T08:29:38,654 DEBUG [RS:1;30c28c82771d:40241 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-24T08:29:38,661 DEBUG [RS:2;30c28c82771d:38023 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-24T08:29:38,662 DEBUG [RS:0;30c28c82771d:40117 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-24T08:29:38,662 DEBUG [RS:1;30c28c82771d:40241 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-24T08:29:38,662 DEBUG [RS:2;30c28c82771d:38023 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-24T08:29:38,662 DEBUG [RS:0;30c28c82771d:40117 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-24T08:29:38,662 DEBUG [RS:1;30c28c82771d:40241 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-24T08:29:38,665 DEBUG [RS:2;30c28c82771d:38023 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-24T08:29:38,665 DEBUG [RS:0;30c28c82771d:40117 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-24T08:29:38,665 DEBUG [RS:1;30c28c82771d:40241 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-24T08:29:38,666 DEBUG [RS:0;30c28c82771d:40117 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@76f2408e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=30c28c82771d/172.17.0.2:0 2024-11-24T08:29:38,666 DEBUG [RS:2;30c28c82771d:38023 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@25b4af86, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=30c28c82771d/172.17.0.2:0 2024-11-24T08:29:38,666 DEBUG [RS:1;30c28c82771d:40241 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5d6cd12a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=30c28c82771d/172.17.0.2:0 2024-11-24T08:29:38,679 DEBUG [master/30c28c82771d:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-24T08:29:38,687 DEBUG [RS:2;30c28c82771d:38023 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:2;30c28c82771d:38023 2024-11-24T08:29:38,689 DEBUG [RS:1;30c28c82771d:40241 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:1;30c28c82771d:40241 2024-11-24T08:29:38,691 DEBUG [RS:0;30c28c82771d:40117 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;30c28c82771d:40117 2024-11-24T08:29:38,691 INFO [master/30c28c82771d:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2 2024-11-24T08:29:38,691 INFO [RS:2;30c28c82771d:38023 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-24T08:29:38,691 INFO [RS:1;30c28c82771d:40241 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-24T08:29:38,691 INFO [RS:0;30c28c82771d:40117 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-24T08:29:38,692 INFO [RS:1;30c28c82771d:40241 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-24T08:29:38,692 INFO [RS:2;30c28c82771d:38023 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-24T08:29:38,692 INFO [RS:0;30c28c82771d:40117 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-24T08:29:38,692 DEBUG [RS:2;30c28c82771d:38023 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-24T08:29:38,692 DEBUG [RS:1;30c28c82771d:40241 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-24T08:29:38,692 DEBUG [RS:0;30c28c82771d:40117 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-24T08:29:38,695 INFO [RS:0;30c28c82771d:40117 {}] regionserver.HRegionServer(2659): reportForDuty to master=30c28c82771d,37719,1732436976835 with port=40117, startcode=1732436977557 2024-11-24T08:29:38,695 INFO [RS:2;30c28c82771d:38023 {}] regionserver.HRegionServer(2659): reportForDuty to master=30c28c82771d,37719,1732436976835 with port=38023, startcode=1732436977716 2024-11-24T08:29:38,695 INFO [RS:1;30c28c82771d:40241 {}] regionserver.HRegionServer(2659): reportForDuty to master=30c28c82771d,37719,1732436976835 with port=40241, startcode=1732436977670 2024-11-24T08:29:38,699 INFO [master/30c28c82771d:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc. 2024-11-24T08:29:38,706 DEBUG [master/30c28c82771d:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: 30c28c82771d,37719,1732436976835 Number of backup masters: 0 Number of live region servers: 0 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2024-11-24T08:29:38,709 DEBUG [RS:0;30c28c82771d:40117 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-24T08:29:38,709 DEBUG [RS:1;30c28c82771d:40241 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-24T08:29:38,709 DEBUG [RS:2;30c28c82771d:38023 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-24T08:29:38,713 DEBUG [master/30c28c82771d:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/30c28c82771d:0, corePoolSize=5, maxPoolSize=5 2024-11-24T08:29:38,713 DEBUG [master/30c28c82771d:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/30c28c82771d:0, corePoolSize=5, maxPoolSize=5 2024-11-24T08:29:38,713 DEBUG [master/30c28c82771d:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/30c28c82771d:0, corePoolSize=5, maxPoolSize=5 2024-11-24T08:29:38,714 DEBUG [master/30c28c82771d:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/30c28c82771d:0, corePoolSize=5, maxPoolSize=5 2024-11-24T08:29:38,714 DEBUG [master/30c28c82771d:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/30c28c82771d:0, corePoolSize=10, maxPoolSize=10 2024-11-24T08:29:38,714 DEBUG [master/30c28c82771d:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/30c28c82771d:0, corePoolSize=1, maxPoolSize=1 2024-11-24T08:29:38,714 DEBUG [master/30c28c82771d:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/30c28c82771d:0, corePoolSize=2, maxPoolSize=2 2024-11-24T08:29:38,714 DEBUG [master/30c28c82771d:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/30c28c82771d:0, corePoolSize=1, maxPoolSize=1 2024-11-24T08:29:38,717 INFO [master/30c28c82771d:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732437008716 2024-11-24T08:29:38,719 INFO [master/30c28c82771d:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2024-11-24T08:29:38,720 INFO [master/30c28c82771d:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2024-11-24T08:29:38,721 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-24T08:29:38,722 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region 2024-11-24T08:29:38,725 INFO [master/30c28c82771d:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2024-11-24T08:29:38,725 INFO [master/30c28c82771d:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2024-11-24T08:29:38,725 INFO [master/30c28c82771d:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2024-11-24T08:29:38,726 INFO [master/30c28c82771d:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2024-11-24T08:29:38,729 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:29:38,729 INFO [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2024-11-24T08:29:38,728 INFO [master/30c28c82771d:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-24T08:29:38,732 INFO [master/30c28c82771d:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2024-11-24T08:29:38,734 INFO [master/30c28c82771d:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2024-11-24T08:29:38,734 INFO [master/30c28c82771d:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2024-11-24T08:29:38,740 INFO [master/30c28c82771d:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2024-11-24T08:29:38,742 INFO [master/30c28c82771d:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2024-11-24T08:29:38,746 DEBUG [master/30c28c82771d:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/30c28c82771d:0:becomeActiveMaster-HFileCleaner.large.0-1732436978744,5,FailOnTimeoutGroup] 2024-11-24T08:29:38,748 DEBUG [master/30c28c82771d:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/30c28c82771d:0:becomeActiveMaster-HFileCleaner.small.0-1732436978746,5,FailOnTimeoutGroup] 2024-11-24T08:29:38,748 INFO [master/30c28c82771d:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-24T08:29:38,749 INFO [master/30c28c82771d:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2024-11-24T08:29:38,778 INFO [master/30c28c82771d:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2024-11-24T08:29:38,779 INFO [master/30c28c82771d:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2024-11-24T08:29:38,780 INFO [HMaster-EventLoopGroup-2-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58295, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=RegionServerStatusService 2024-11-24T08:29:38,780 INFO [HMaster-EventLoopGroup-2-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49427, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins.hfs.2 (auth:SIMPLE), service=RegionServerStatusService 2024-11-24T08:29:38,780 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40533 is added to blk_1073741831_1007 (size=1321) 2024-11-24T08:29:38,780 INFO [HMaster-EventLoopGroup-2-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59153, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins.hfs.1 (auth:SIMPLE), service=RegionServerStatusService 2024-11-24T08:29:38,781 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34459 is added to blk_1073741831_1007 (size=1321) 2024-11-24T08:29:38,781 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33555 is added to blk_1073741831_1007 (size=1321) 2024-11-24T08:29:38,784 INFO [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to hdfs://localhost:46165/user/jenkins/test-data/7507fec3-e300-7585-3a43-9c9d3ed5b09e/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321 2024-11-24T08:29:38,784 INFO [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:46165/user/jenkins/test-data/7507fec3-e300-7585-3a43-9c9d3ed5b09e 2024-11-24T08:29:38,786 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=37719 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 30c28c82771d,40117,1732436977557 2024-11-24T08:29:38,789 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=37719 {}] master.ServerManager(517): Registering regionserver=30c28c82771d,40117,1732436977557 2024-11-24T08:29:38,801 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40533 is added to blk_1073741832_1008 (size=32) 2024-11-24T08:29:38,802 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34459 is added to blk_1073741832_1008 (size=32) 2024-11-24T08:29:38,802 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33555 is added to blk_1073741832_1008 (size=32) 2024-11-24T08:29:38,803 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=37719 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 30c28c82771d,38023,1732436977716 2024-11-24T08:29:38,803 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=37719 {}] master.ServerManager(517): Registering regionserver=30c28c82771d,38023,1732436977716 2024-11-24T08:29:38,808 DEBUG [RS:0;30c28c82771d:40117 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:46165/user/jenkins/test-data/7507fec3-e300-7585-3a43-9c9d3ed5b09e 2024-11-24T08:29:38,808 DEBUG [RS:0;30c28c82771d:40117 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:46165 2024-11-24T08:29:38,808 DEBUG [RS:0;30c28c82771d:40117 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-24T08:29:38,810 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=37719 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 30c28c82771d,40241,1732436977670 2024-11-24T08:29:38,810 DEBUG [RS:2;30c28c82771d:38023 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:46165/user/jenkins/test-data/7507fec3-e300-7585-3a43-9c9d3ed5b09e 2024-11-24T08:29:38,810 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=37719 {}] master.ServerManager(517): Registering regionserver=30c28c82771d,40241,1732436977670 2024-11-24T08:29:38,810 DEBUG [RS:2;30c28c82771d:38023 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:46165 2024-11-24T08:29:38,810 DEBUG [RS:2;30c28c82771d:38023 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-24T08:29:38,813 DEBUG [RS:1;30c28c82771d:40241 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:46165/user/jenkins/test-data/7507fec3-e300-7585-3a43-9c9d3ed5b09e 2024-11-24T08:29:38,814 DEBUG [RS:1;30c28c82771d:40241 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:46165 2024-11-24T08:29:38,814 DEBUG [RS:1;30c28c82771d:40241 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-24T08:29:38,816 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37719-0x10149184c260000, quorum=127.0.0.1:59068, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-24T08:29:38,817 DEBUG [RS:0;30c28c82771d:40117 {}] zookeeper.ZKUtil(111): regionserver:40117-0x10149184c260001, quorum=127.0.0.1:59068, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/30c28c82771d,40117,1732436977557 2024-11-24T08:29:38,818 DEBUG [RS:2;30c28c82771d:38023 {}] zookeeper.ZKUtil(111): regionserver:38023-0x10149184c260003, quorum=127.0.0.1:59068, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/30c28c82771d,38023,1732436977716 2024-11-24T08:29:38,818 WARN [RS:0;30c28c82771d:40117 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-24T08:29:38,818 WARN [RS:2;30c28c82771d:38023 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-24T08:29:38,818 INFO [RS:0;30c28c82771d:40117 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-24T08:29:38,818 INFO [RS:2;30c28c82771d:38023 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-24T08:29:38,818 DEBUG [RS:0;30c28c82771d:40117 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:46165/user/jenkins/test-data/7507fec3-e300-7585-3a43-9c9d3ed5b09e/WALs/30c28c82771d,40117,1732436977557 2024-11-24T08:29:38,818 DEBUG [RS:2;30c28c82771d:38023 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:46165/user/jenkins/test-data/7507fec3-e300-7585-3a43-9c9d3ed5b09e/WALs/30c28c82771d,38023,1732436977716 2024-11-24T08:29:38,819 DEBUG [RS:1;30c28c82771d:40241 {}] zookeeper.ZKUtil(111): regionserver:40241-0x10149184c260002, quorum=127.0.0.1:59068, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/30c28c82771d,40241,1732436977670 2024-11-24T08:29:38,819 WARN [RS:1;30c28c82771d:40241 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-24T08:29:38,819 INFO [RS:1;30c28c82771d:40241 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-24T08:29:38,819 DEBUG [RS:1;30c28c82771d:40241 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:46165/user/jenkins/test-data/7507fec3-e300-7585-3a43-9c9d3ed5b09e/WALs/30c28c82771d,40241,1732436977670 2024-11-24T08:29:38,821 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [30c28c82771d,40241,1732436977670] 2024-11-24T08:29:38,821 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [30c28c82771d,38023,1732436977716] 2024-11-24T08:29:38,821 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [30c28c82771d,40117,1732436977557] 2024-11-24T08:29:38,847 INFO [RS:1;30c28c82771d:40241 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-24T08:29:38,847 INFO [RS:2;30c28c82771d:38023 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-24T08:29:38,847 INFO [RS:0;30c28c82771d:40117 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-24T08:29:38,860 INFO [RS:1;30c28c82771d:40241 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-24T08:29:38,861 INFO [RS:0;30c28c82771d:40117 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-24T08:29:38,861 INFO [RS:2;30c28c82771d:38023 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-24T08:29:38,866 INFO [RS:0;30c28c82771d:40117 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-24T08:29:38,866 INFO [RS:2;30c28c82771d:38023 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-24T08:29:38,866 INFO [RS:1;30c28c82771d:40241 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-24T08:29:38,866 INFO [RS:2;30c28c82771d:38023 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-24T08:29:38,866 INFO [RS:0;30c28c82771d:40117 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-24T08:29:38,866 INFO [RS:1;30c28c82771d:40241 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-24T08:29:38,867 INFO [RS:2;30c28c82771d:38023 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-24T08:29:38,868 INFO [RS:0;30c28c82771d:40117 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-24T08:29:38,868 INFO [RS:1;30c28c82771d:40241 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-24T08:29:38,874 INFO [RS:0;30c28c82771d:40117 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-24T08:29:38,874 INFO [RS:2;30c28c82771d:38023 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-24T08:29:38,875 INFO [RS:0;30c28c82771d:40117 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-24T08:29:38,875 INFO [RS:2;30c28c82771d:38023 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-24T08:29:38,875 INFO [RS:1;30c28c82771d:40241 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-24T08:29:38,876 DEBUG [RS:2;30c28c82771d:38023 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/30c28c82771d:0, corePoolSize=1, maxPoolSize=1 2024-11-24T08:29:38,876 DEBUG [RS:0;30c28c82771d:40117 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/30c28c82771d:0, corePoolSize=1, maxPoolSize=1 2024-11-24T08:29:38,876 INFO [RS:1;30c28c82771d:40241 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-24T08:29:38,876 DEBUG [RS:0;30c28c82771d:40117 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/30c28c82771d:0, corePoolSize=1, maxPoolSize=1 2024-11-24T08:29:38,876 DEBUG [RS:2;30c28c82771d:38023 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/30c28c82771d:0, corePoolSize=1, maxPoolSize=1 2024-11-24T08:29:38,876 DEBUG [RS:1;30c28c82771d:40241 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/30c28c82771d:0, corePoolSize=1, maxPoolSize=1 2024-11-24T08:29:38,876 DEBUG [RS:0;30c28c82771d:40117 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/30c28c82771d:0, corePoolSize=1, maxPoolSize=1 2024-11-24T08:29:38,876 DEBUG [RS:2;30c28c82771d:38023 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/30c28c82771d:0, corePoolSize=1, maxPoolSize=1 2024-11-24T08:29:38,876 DEBUG [RS:1;30c28c82771d:40241 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/30c28c82771d:0, corePoolSize=1, maxPoolSize=1 2024-11-24T08:29:38,876 DEBUG [RS:2;30c28c82771d:38023 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/30c28c82771d:0, corePoolSize=1, maxPoolSize=1 2024-11-24T08:29:38,876 DEBUG [RS:0;30c28c82771d:40117 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/30c28c82771d:0, corePoolSize=1, maxPoolSize=1 2024-11-24T08:29:38,877 DEBUG [RS:1;30c28c82771d:40241 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/30c28c82771d:0, corePoolSize=1, maxPoolSize=1 2024-11-24T08:29:38,877 DEBUG [RS:0;30c28c82771d:40117 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/30c28c82771d:0, corePoolSize=1, maxPoolSize=1 2024-11-24T08:29:38,877 DEBUG [RS:2;30c28c82771d:38023 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/30c28c82771d:0, corePoolSize=1, maxPoolSize=1 2024-11-24T08:29:38,877 DEBUG [RS:1;30c28c82771d:40241 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/30c28c82771d:0, corePoolSize=1, maxPoolSize=1 2024-11-24T08:29:38,877 DEBUG [RS:0;30c28c82771d:40117 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/30c28c82771d:0, corePoolSize=2, maxPoolSize=2 2024-11-24T08:29:38,877 DEBUG [RS:2;30c28c82771d:38023 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/30c28c82771d:0, corePoolSize=2, maxPoolSize=2 2024-11-24T08:29:38,877 DEBUG [RS:1;30c28c82771d:40241 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/30c28c82771d:0, corePoolSize=1, maxPoolSize=1 2024-11-24T08:29:38,877 DEBUG [RS:0;30c28c82771d:40117 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/30c28c82771d:0, corePoolSize=1, maxPoolSize=1 2024-11-24T08:29:38,877 DEBUG [RS:2;30c28c82771d:38023 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/30c28c82771d:0, corePoolSize=1, maxPoolSize=1 2024-11-24T08:29:38,877 DEBUG [RS:1;30c28c82771d:40241 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/30c28c82771d:0, corePoolSize=2, maxPoolSize=2 2024-11-24T08:29:38,877 DEBUG [RS:0;30c28c82771d:40117 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/30c28c82771d:0, corePoolSize=1, maxPoolSize=1 2024-11-24T08:29:38,877 DEBUG [RS:2;30c28c82771d:38023 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/30c28c82771d:0, corePoolSize=1, maxPoolSize=1 2024-11-24T08:29:38,877 DEBUG [RS:0;30c28c82771d:40117 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/30c28c82771d:0, corePoolSize=1, maxPoolSize=1 2024-11-24T08:29:38,877 DEBUG [RS:1;30c28c82771d:40241 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/30c28c82771d:0, corePoolSize=1, maxPoolSize=1 2024-11-24T08:29:38,877 DEBUG [RS:0;30c28c82771d:40117 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/30c28c82771d:0, corePoolSize=1, maxPoolSize=1 2024-11-24T08:29:38,877 DEBUG [RS:2;30c28c82771d:38023 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/30c28c82771d:0, corePoolSize=1, maxPoolSize=1 2024-11-24T08:29:38,878 DEBUG [RS:1;30c28c82771d:40241 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/30c28c82771d:0, corePoolSize=1, maxPoolSize=1 2024-11-24T08:29:38,878 DEBUG [RS:0;30c28c82771d:40117 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/30c28c82771d:0, corePoolSize=1, maxPoolSize=1 2024-11-24T08:29:38,878 DEBUG [RS:2;30c28c82771d:38023 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/30c28c82771d:0, corePoolSize=1, maxPoolSize=1 2024-11-24T08:29:38,878 DEBUG [RS:0;30c28c82771d:40117 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/30c28c82771d:0, corePoolSize=1, maxPoolSize=1 2024-11-24T08:29:38,878 DEBUG [RS:1;30c28c82771d:40241 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/30c28c82771d:0, corePoolSize=1, maxPoolSize=1 2024-11-24T08:29:38,878 DEBUG [RS:2;30c28c82771d:38023 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/30c28c82771d:0, corePoolSize=1, maxPoolSize=1 2024-11-24T08:29:38,878 DEBUG [RS:0;30c28c82771d:40117 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/30c28c82771d:0, corePoolSize=3, maxPoolSize=3 2024-11-24T08:29:38,878 DEBUG [RS:1;30c28c82771d:40241 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/30c28c82771d:0, corePoolSize=1, maxPoolSize=1 2024-11-24T08:29:38,878 DEBUG [RS:0;30c28c82771d:40117 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0, corePoolSize=3, maxPoolSize=3 2024-11-24T08:29:38,878 DEBUG [RS:2;30c28c82771d:38023 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/30c28c82771d:0, corePoolSize=1, maxPoolSize=1 2024-11-24T08:29:38,878 DEBUG [RS:1;30c28c82771d:40241 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/30c28c82771d:0, corePoolSize=1, maxPoolSize=1 2024-11-24T08:29:38,878 DEBUG [RS:2;30c28c82771d:38023 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/30c28c82771d:0, corePoolSize=3, maxPoolSize=3 2024-11-24T08:29:38,878 DEBUG [RS:1;30c28c82771d:40241 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/30c28c82771d:0, corePoolSize=1, maxPoolSize=1 2024-11-24T08:29:38,878 DEBUG [RS:2;30c28c82771d:38023 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0, corePoolSize=3, maxPoolSize=3 2024-11-24T08:29:38,879 DEBUG [RS:1;30c28c82771d:40241 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/30c28c82771d:0, corePoolSize=3, maxPoolSize=3 2024-11-24T08:29:38,879 DEBUG [RS:1;30c28c82771d:40241 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0, corePoolSize=3, maxPoolSize=3 2024-11-24T08:29:38,880 INFO [RS:0;30c28c82771d:40117 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-24T08:29:38,881 INFO [RS:0;30c28c82771d:40117 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-24T08:29:38,881 INFO [RS:0;30c28c82771d:40117 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-24T08:29:38,881 INFO [RS:0;30c28c82771d:40117 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-24T08:29:38,881 INFO [RS:0;30c28c82771d:40117 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-24T08:29:38,881 INFO [RS:0;30c28c82771d:40117 {}] hbase.ChoreService(168): Chore ScheduledChore name=30c28c82771d,40117,1732436977557-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-24T08:29:38,881 INFO [RS:2;30c28c82771d:38023 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-24T08:29:38,881 INFO [RS:2;30c28c82771d:38023 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-24T08:29:38,882 INFO [RS:1;30c28c82771d:40241 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-24T08:29:38,882 INFO [RS:2;30c28c82771d:38023 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-24T08:29:38,882 INFO [RS:1;30c28c82771d:40241 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-24T08:29:38,882 INFO [RS:2;30c28c82771d:38023 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-24T08:29:38,882 INFO [RS:1;30c28c82771d:40241 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-24T08:29:38,882 INFO [RS:2;30c28c82771d:38023 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-24T08:29:38,882 INFO [RS:1;30c28c82771d:40241 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-24T08:29:38,882 INFO [RS:2;30c28c82771d:38023 {}] hbase.ChoreService(168): Chore ScheduledChore name=30c28c82771d,38023,1732436977716-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-24T08:29:38,882 INFO [RS:1;30c28c82771d:40241 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-24T08:29:38,882 INFO [RS:1;30c28c82771d:40241 {}] hbase.ChoreService(168): Chore ScheduledChore name=30c28c82771d,40241,1732436977670-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-24T08:29:38,902 INFO [RS:2;30c28c82771d:38023 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-24T08:29:38,904 INFO [RS:0;30c28c82771d:40117 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-24T08:29:38,905 INFO [RS:2;30c28c82771d:38023 {}] hbase.ChoreService(168): Chore ScheduledChore name=30c28c82771d,38023,1732436977716-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-24T08:29:38,905 INFO [RS:0;30c28c82771d:40117 {}] hbase.ChoreService(168): Chore ScheduledChore name=30c28c82771d,40117,1732436977557-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-24T08:29:38,905 INFO [RS:0;30c28c82771d:40117 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-24T08:29:38,905 INFO [RS:0;30c28c82771d:40117 {}] regionserver.Replication(171): 30c28c82771d,40117,1732436977557 started 2024-11-24T08:29:38,906 INFO [RS:2;30c28c82771d:38023 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-24T08:29:38,906 INFO [RS:2;30c28c82771d:38023 {}] regionserver.Replication(171): 30c28c82771d,38023,1732436977716 started 2024-11-24T08:29:38,906 INFO [RS:1;30c28c82771d:40241 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-24T08:29:38,906 INFO [RS:1;30c28c82771d:40241 {}] hbase.ChoreService(168): Chore ScheduledChore name=30c28c82771d,40241,1732436977670-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-24T08:29:38,907 INFO [RS:1;30c28c82771d:40241 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-24T08:29:38,907 INFO [RS:1;30c28c82771d:40241 {}] regionserver.Replication(171): 30c28c82771d,40241,1732436977670 started 2024-11-24T08:29:38,925 INFO [RS:2;30c28c82771d:38023 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-24T08:29:38,925 INFO [RS:2;30c28c82771d:38023 {}] regionserver.HRegionServer(1482): Serving as 30c28c82771d,38023,1732436977716, RpcServer on 30c28c82771d/172.17.0.2:38023, sessionid=0x10149184c260003 2024-11-24T08:29:38,926 DEBUG [RS:2;30c28c82771d:38023 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-24T08:29:38,926 DEBUG [RS:2;30c28c82771d:38023 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 30c28c82771d,38023,1732436977716 2024-11-24T08:29:38,926 DEBUG [RS:2;30c28c82771d:38023 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '30c28c82771d,38023,1732436977716' 2024-11-24T08:29:38,927 DEBUG [RS:2;30c28c82771d:38023 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-24T08:29:38,928 DEBUG [RS:2;30c28c82771d:38023 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-24T08:29:38,928 DEBUG [RS:2;30c28c82771d:38023 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-24T08:29:38,928 DEBUG [RS:2;30c28c82771d:38023 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-24T08:29:38,929 DEBUG [RS:2;30c28c82771d:38023 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 30c28c82771d,38023,1732436977716 2024-11-24T08:29:38,929 DEBUG [RS:2;30c28c82771d:38023 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '30c28c82771d,38023,1732436977716' 2024-11-24T08:29:38,929 DEBUG [RS:2;30c28c82771d:38023 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-24T08:29:38,929 DEBUG [RS:2;30c28c82771d:38023 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-24T08:29:38,930 DEBUG [RS:2;30c28c82771d:38023 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-24T08:29:38,930 INFO [RS:2;30c28c82771d:38023 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-24T08:29:38,930 INFO [RS:2;30c28c82771d:38023 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-24T08:29:38,931 INFO [RS:0;30c28c82771d:40117 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-24T08:29:38,931 INFO [RS:1;30c28c82771d:40241 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-24T08:29:38,931 INFO [RS:0;30c28c82771d:40117 {}] regionserver.HRegionServer(1482): Serving as 30c28c82771d,40117,1732436977557, RpcServer on 30c28c82771d/172.17.0.2:40117, sessionid=0x10149184c260001 2024-11-24T08:29:38,931 INFO [RS:1;30c28c82771d:40241 {}] regionserver.HRegionServer(1482): Serving as 30c28c82771d,40241,1732436977670, RpcServer on 30c28c82771d/172.17.0.2:40241, sessionid=0x10149184c260002 2024-11-24T08:29:38,931 DEBUG [RS:0;30c28c82771d:40117 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-24T08:29:38,931 DEBUG [RS:1;30c28c82771d:40241 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-24T08:29:38,931 DEBUG [RS:0;30c28c82771d:40117 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 30c28c82771d,40117,1732436977557 2024-11-24T08:29:38,931 DEBUG [RS:1;30c28c82771d:40241 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 30c28c82771d,40241,1732436977670 2024-11-24T08:29:38,931 DEBUG [RS:1;30c28c82771d:40241 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '30c28c82771d,40241,1732436977670' 2024-11-24T08:29:38,931 DEBUG [RS:0;30c28c82771d:40117 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '30c28c82771d,40117,1732436977557' 2024-11-24T08:29:38,931 DEBUG [RS:1;30c28c82771d:40241 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-24T08:29:38,931 DEBUG [RS:0;30c28c82771d:40117 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-24T08:29:38,932 DEBUG [RS:0;30c28c82771d:40117 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-24T08:29:38,932 DEBUG [RS:1;30c28c82771d:40241 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-24T08:29:38,933 DEBUG [RS:0;30c28c82771d:40117 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-24T08:29:38,933 DEBUG [RS:1;30c28c82771d:40241 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-24T08:29:38,933 DEBUG [RS:0;30c28c82771d:40117 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-24T08:29:38,933 DEBUG [RS:1;30c28c82771d:40241 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-24T08:29:38,933 DEBUG [RS:0;30c28c82771d:40117 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 30c28c82771d,40117,1732436977557 2024-11-24T08:29:38,933 DEBUG [RS:1;30c28c82771d:40241 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 30c28c82771d,40241,1732436977670 2024-11-24T08:29:38,933 DEBUG [RS:0;30c28c82771d:40117 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '30c28c82771d,40117,1732436977557' 2024-11-24T08:29:38,933 DEBUG [RS:1;30c28c82771d:40241 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '30c28c82771d,40241,1732436977670' 2024-11-24T08:29:38,933 DEBUG [RS:0;30c28c82771d:40117 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-24T08:29:38,933 DEBUG [RS:1;30c28c82771d:40241 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-24T08:29:38,934 DEBUG [RS:0;30c28c82771d:40117 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-24T08:29:38,934 DEBUG [RS:1;30c28c82771d:40241 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-24T08:29:38,934 DEBUG [RS:1;30c28c82771d:40241 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-24T08:29:38,934 DEBUG [RS:0;30c28c82771d:40117 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-24T08:29:38,934 INFO [RS:1;30c28c82771d:40241 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-24T08:29:38,934 INFO [RS:1;30c28c82771d:40241 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-24T08:29:38,934 INFO [RS:0;30c28c82771d:40117 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-24T08:29:38,934 INFO [RS:0;30c28c82771d:40117 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-24T08:29:39,036 INFO [RS:0;30c28c82771d:40117 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-24T08:29:39,036 INFO [RS:2;30c28c82771d:38023 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-24T08:29:39,036 INFO [RS:1;30c28c82771d:40241 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-24T08:29:39,040 INFO [RS:2;30c28c82771d:38023 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=30c28c82771d%2C38023%2C1732436977716, suffix=, logDir=hdfs://localhost:46165/user/jenkins/test-data/7507fec3-e300-7585-3a43-9c9d3ed5b09e/WALs/30c28c82771d,38023,1732436977716, archiveDir=hdfs://localhost:46165/user/jenkins/test-data/7507fec3-e300-7585-3a43-9c9d3ed5b09e/oldWALs, maxLogs=32 2024-11-24T08:29:39,040 INFO [RS:0;30c28c82771d:40117 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=30c28c82771d%2C40117%2C1732436977557, suffix=, logDir=hdfs://localhost:46165/user/jenkins/test-data/7507fec3-e300-7585-3a43-9c9d3ed5b09e/WALs/30c28c82771d,40117,1732436977557, archiveDir=hdfs://localhost:46165/user/jenkins/test-data/7507fec3-e300-7585-3a43-9c9d3ed5b09e/oldWALs, maxLogs=32 2024-11-24T08:29:39,040 INFO [RS:1;30c28c82771d:40241 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=30c28c82771d%2C40241%2C1732436977670, suffix=, logDir=hdfs://localhost:46165/user/jenkins/test-data/7507fec3-e300-7585-3a43-9c9d3ed5b09e/WALs/30c28c82771d,40241,1732436977670, archiveDir=hdfs://localhost:46165/user/jenkins/test-data/7507fec3-e300-7585-3a43-9c9d3ed5b09e/oldWALs, maxLogs=32 2024-11-24T08:29:39,061 DEBUG [RS:2;30c28c82771d:38023 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/7507fec3-e300-7585-3a43-9c9d3ed5b09e/WALs/30c28c82771d,38023,1732436977716/30c28c82771d%2C38023%2C1732436977716.1732436979045, exclude list is [], retry=0 2024-11-24T08:29:39,061 DEBUG [RS:1;30c28c82771d:40241 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/7507fec3-e300-7585-3a43-9c9d3ed5b09e/WALs/30c28c82771d,40241,1732436977670/30c28c82771d%2C40241%2C1732436977670.1732436979045, exclude list is [], retry=0 2024-11-24T08:29:39,061 DEBUG [RS:0;30c28c82771d:40117 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/7507fec3-e300-7585-3a43-9c9d3ed5b09e/WALs/30c28c82771d,40117,1732436977557/30c28c82771d%2C40117%2C1732436977557.1732436979045, exclude list is [], retry=0 2024-11-24T08:29:39,067 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33555,DS-8d97c3e2-1525-418c-a970-00a7157a0d31,DISK] 2024-11-24T08:29:39,067 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33555,DS-8d97c3e2-1525-418c-a970-00a7157a0d31,DISK] 2024-11-24T08:29:39,067 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:34459,DS-6b4d1656-f49b-4a16-be49-2eb682c16e11,DISK] 2024-11-24T08:29:39,067 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:34459,DS-6b4d1656-f49b-4a16-be49-2eb682c16e11,DISK] 2024-11-24T08:29:39,068 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40533,DS-5eeb9666-6dd5-4a48-9fb1-3f9aff1acdca,DISK] 2024-11-24T08:29:39,068 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40533,DS-5eeb9666-6dd5-4a48-9fb1-3f9aff1acdca,DISK] 2024-11-24T08:29:39,070 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40533,DS-5eeb9666-6dd5-4a48-9fb1-3f9aff1acdca,DISK] 2024-11-24T08:29:39,070 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:34459,DS-6b4d1656-f49b-4a16-be49-2eb682c16e11,DISK] 2024-11-24T08:29:39,071 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33555,DS-8d97c3e2-1525-418c-a970-00a7157a0d31,DISK] 2024-11-24T08:29:39,083 INFO [RS:2;30c28c82771d:38023 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/7507fec3-e300-7585-3a43-9c9d3ed5b09e/WALs/30c28c82771d,38023,1732436977716/30c28c82771d%2C38023%2C1732436977716.1732436979045 2024-11-24T08:29:39,086 DEBUG [RS:2;30c28c82771d:38023 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:39357:39357),(127.0.0.1/127.0.0.1:44131:44131),(127.0.0.1/127.0.0.1:42673:42673)] 2024-11-24T08:29:39,088 INFO [RS:0;30c28c82771d:40117 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/7507fec3-e300-7585-3a43-9c9d3ed5b09e/WALs/30c28c82771d,40117,1732436977557/30c28c82771d%2C40117%2C1732436977557.1732436979045 2024-11-24T08:29:39,089 DEBUG [RS:0;30c28c82771d:40117 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:39357:39357),(127.0.0.1/127.0.0.1:42673:42673),(127.0.0.1/127.0.0.1:44131:44131)] 2024-11-24T08:29:39,091 INFO [RS:1;30c28c82771d:40241 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/7507fec3-e300-7585-3a43-9c9d3ed5b09e/WALs/30c28c82771d,40241,1732436977670/30c28c82771d%2C40241%2C1732436977670.1732436979045 2024-11-24T08:29:39,091 DEBUG [RS:1;30c28c82771d:40241 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:39357:39357),(127.0.0.1/127.0.0.1:44131:44131),(127.0.0.1/127.0.0.1:42673:42673)] 2024-11-24T08:29:39,203 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T08:29:39,206 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-24T08:29:39,210 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-24T08:29:39,210 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:29:39,211 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-24T08:29:39,211 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-24T08:29:39,214 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-24T08:29:39,214 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:29:39,215 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-24T08:29:39,215 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-24T08:29:39,218 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-24T08:29:39,218 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:29:39,219 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-24T08:29:39,219 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-24T08:29:39,222 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-24T08:29:39,222 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:29:39,223 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-24T08:29:39,223 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-24T08:29:39,224 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46165/user/jenkins/test-data/7507fec3-e300-7585-3a43-9c9d3ed5b09e/data/hbase/meta/1588230740 2024-11-24T08:29:39,225 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46165/user/jenkins/test-data/7507fec3-e300-7585-3a43-9c9d3ed5b09e/data/hbase/meta/1588230740 2024-11-24T08:29:39,228 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-24T08:29:39,228 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-24T08:29:39,230 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-24T08:29:39,232 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-24T08:29:39,236 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:46165/user/jenkins/test-data/7507fec3-e300-7585-3a43-9c9d3ed5b09e/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-24T08:29:39,237 INFO [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=62446910, jitterRate=-0.06946852803230286}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-24T08:29:39,239 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740: Writing region info on filesystem at 1732436979203Initializing all the Stores at 1732436979205 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732436979205Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732436979206 (+1 ms)Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732436979206Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732436979206Cleaning up temporary data from old regions at 1732436979228 (+22 ms)Region opened successfully at 1732436979239 (+11 ms) 2024-11-24T08:29:39,239 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-24T08:29:39,240 INFO [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-24T08:29:39,240 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-24T08:29:39,240 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-24T08:29:39,240 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-24T08:29:39,241 INFO [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-24T08:29:39,242 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1732436979239Disabling compacts and flushes for region at 1732436979239Disabling writes for close at 1732436979240 (+1 ms)Writing region close event to WAL at 1732436979241 (+1 ms)Closed at 1732436979241 2024-11-24T08:29:39,245 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-24T08:29:39,245 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta 2024-11-24T08:29:39,253 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-11-24T08:29:39,264 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-24T08:29:39,268 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2024-11-24T08:29:39,420 DEBUG [30c28c82771d:37719 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=3, allServersCount=3 2024-11-24T08:29:39,429 DEBUG [30c28c82771d:37719 {}] balancer.BalancerClusterState(204): Hosts are {30c28c82771d=0} racks are {/default-rack=0} 2024-11-24T08:29:39,437 DEBUG [30c28c82771d:37719 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-11-24T08:29:39,437 DEBUG [30c28c82771d:37719 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-11-24T08:29:39,437 DEBUG [30c28c82771d:37719 {}] balancer.BalancerClusterState(303): server 2 has 0 regions 2024-11-24T08:29:39,437 DEBUG [30c28c82771d:37719 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-11-24T08:29:39,437 DEBUG [30c28c82771d:37719 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-11-24T08:29:39,437 DEBUG [30c28c82771d:37719 {}] balancer.BalancerClusterState(310): server 2 is on host 0 2024-11-24T08:29:39,437 INFO [30c28c82771d:37719 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-11-24T08:29:39,437 INFO [30c28c82771d:37719 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-11-24T08:29:39,437 INFO [30c28c82771d:37719 {}] balancer.BalancerClusterState(321): server 2 is on rack 0 2024-11-24T08:29:39,437 DEBUG [30c28c82771d:37719 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-11-24T08:29:39,447 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=30c28c82771d,40241,1732436977670 2024-11-24T08:29:39,452 INFO [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 30c28c82771d,40241,1732436977670, state=OPENING 2024-11-24T08:29:39,458 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it 2024-11-24T08:29:39,460 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37719-0x10149184c260000, quorum=127.0.0.1:59068, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-24T08:29:39,460 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40241-0x10149184c260002, quorum=127.0.0.1:59068, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-24T08:29:39,460 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40117-0x10149184c260001, quorum=127.0.0.1:59068, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-24T08:29:39,460 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38023-0x10149184c260003, quorum=127.0.0.1:59068, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-24T08:29:39,461 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-24T08:29:39,461 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-24T08:29:39,461 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-24T08:29:39,462 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-24T08:29:39,464 DEBUG [PEWorker-3 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-24T08:29:39,466 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=30c28c82771d,40241,1732436977670}] 2024-11-24T08:29:39,644 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-24T08:29:39,647 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-4-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49693, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-24T08:29:39,661 INFO [RS_OPEN_META-regionserver/30c28c82771d:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-11-24T08:29:39,662 INFO [RS_OPEN_META-regionserver/30c28c82771d:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-24T08:29:39,662 INFO [RS_OPEN_META-regionserver/30c28c82771d:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-11-24T08:29:39,666 INFO [RS_OPEN_META-regionserver/30c28c82771d:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=30c28c82771d%2C40241%2C1732436977670.meta, suffix=.meta, logDir=hdfs://localhost:46165/user/jenkins/test-data/7507fec3-e300-7585-3a43-9c9d3ed5b09e/WALs/30c28c82771d,40241,1732436977670, archiveDir=hdfs://localhost:46165/user/jenkins/test-data/7507fec3-e300-7585-3a43-9c9d3ed5b09e/oldWALs, maxLogs=32 2024-11-24T08:29:39,682 DEBUG [RS_OPEN_META-regionserver/30c28c82771d:0-0 {event_type=M_RS_OPEN_META, pid=3}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/7507fec3-e300-7585-3a43-9c9d3ed5b09e/WALs/30c28c82771d,40241,1732436977670/30c28c82771d%2C40241%2C1732436977670.meta.1732436979668.meta, exclude list is [], retry=0 2024-11-24T08:29:39,686 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33555,DS-8d97c3e2-1525-418c-a970-00a7157a0d31,DISK] 2024-11-24T08:29:39,686 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40533,DS-5eeb9666-6dd5-4a48-9fb1-3f9aff1acdca,DISK] 2024-11-24T08:29:39,686 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:34459,DS-6b4d1656-f49b-4a16-be49-2eb682c16e11,DISK] 2024-11-24T08:29:39,690 INFO [RS_OPEN_META-regionserver/30c28c82771d:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/7507fec3-e300-7585-3a43-9c9d3ed5b09e/WALs/30c28c82771d,40241,1732436977670/30c28c82771d%2C40241%2C1732436977670.meta.1732436979668.meta 2024-11-24T08:29:39,691 DEBUG [RS_OPEN_META-regionserver/30c28c82771d:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:39357:39357),(127.0.0.1/127.0.0.1:42673:42673),(127.0.0.1/127.0.0.1:44131:44131)] 2024-11-24T08:29:39,691 DEBUG [RS_OPEN_META-regionserver/30c28c82771d:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-11-24T08:29:39,693 DEBUG [RS_OPEN_META-regionserver/30c28c82771d:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-11-24T08:29:39,695 DEBUG [RS_OPEN_META-regionserver/30c28c82771d:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-11-24T08:29:39,700 INFO [RS_OPEN_META-regionserver/30c28c82771d:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-11-24T08:29:39,704 DEBUG [RS_OPEN_META-regionserver/30c28c82771d:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-11-24T08:29:39,705 DEBUG [RS_OPEN_META-regionserver/30c28c82771d:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T08:29:39,705 DEBUG [RS_OPEN_META-regionserver/30c28c82771d:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-11-24T08:29:39,705 DEBUG [RS_OPEN_META-regionserver/30c28c82771d:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-11-24T08:29:39,708 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-24T08:29:39,710 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-24T08:29:39,710 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:29:39,711 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-24T08:29:39,711 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-24T08:29:39,712 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-24T08:29:39,712 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:29:39,713 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-24T08:29:39,713 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-24T08:29:39,714 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-24T08:29:39,714 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:29:39,715 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-24T08:29:39,715 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-24T08:29:39,717 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-24T08:29:39,717 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:29:39,718 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-24T08:29:39,718 DEBUG [RS_OPEN_META-regionserver/30c28c82771d:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-24T08:29:39,719 DEBUG [RS_OPEN_META-regionserver/30c28c82771d:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46165/user/jenkins/test-data/7507fec3-e300-7585-3a43-9c9d3ed5b09e/data/hbase/meta/1588230740 2024-11-24T08:29:39,722 DEBUG [RS_OPEN_META-regionserver/30c28c82771d:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46165/user/jenkins/test-data/7507fec3-e300-7585-3a43-9c9d3ed5b09e/data/hbase/meta/1588230740 2024-11-24T08:29:39,725 DEBUG [RS_OPEN_META-regionserver/30c28c82771d:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-24T08:29:39,725 DEBUG [RS_OPEN_META-regionserver/30c28c82771d:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-24T08:29:39,726 DEBUG [RS_OPEN_META-regionserver/30c28c82771d:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-24T08:29:39,729 DEBUG [RS_OPEN_META-regionserver/30c28c82771d:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-24T08:29:39,731 INFO [RS_OPEN_META-regionserver/30c28c82771d:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=72370252, jitterRate=0.07840079069137573}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-24T08:29:39,731 DEBUG [RS_OPEN_META-regionserver/30c28c82771d:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-11-24T08:29:39,733 DEBUG [RS_OPEN_META-regionserver/30c28c82771d:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1732436979706Writing region info on filesystem at 1732436979706Initializing all the Stores at 1732436979707 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732436979708 (+1 ms)Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732436979708Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732436979708Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732436979708Cleaning up temporary data from old regions at 1732436979725 (+17 ms)Running coprocessor post-open hooks at 1732436979731 (+6 ms)Region opened successfully at 1732436979733 (+2 ms) 2024-11-24T08:29:39,741 INFO [RS_OPEN_META-regionserver/30c28c82771d:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1732436979635 2024-11-24T08:29:39,753 DEBUG [RS_OPEN_META-regionserver/30c28c82771d:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-11-24T08:29:39,753 INFO [RS_OPEN_META-regionserver/30c28c82771d:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-11-24T08:29:39,755 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=30c28c82771d,40241,1732436977670 2024-11-24T08:29:39,758 INFO [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 30c28c82771d,40241,1732436977670, state=OPEN 2024-11-24T08:29:39,760 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38023-0x10149184c260003, quorum=127.0.0.1:59068, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-24T08:29:39,760 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40241-0x10149184c260002, quorum=127.0.0.1:59068, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-24T08:29:39,760 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37719-0x10149184c260000, quorum=127.0.0.1:59068, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-24T08:29:39,760 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-24T08:29:39,760 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-24T08:29:39,760 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40117-0x10149184c260001, quorum=127.0.0.1:59068, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-24T08:29:39,761 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-24T08:29:39,761 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-24T08:29:39,761 DEBUG [PEWorker-5 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=30c28c82771d,40241,1732436977670 2024-11-24T08:29:39,766 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2 2024-11-24T08:29:39,766 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=30c28c82771d,40241,1732436977670 in 295 msec 2024-11-24T08:29:39,774 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1 2024-11-24T08:29:39,774 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 516 msec 2024-11-24T08:29:39,776 DEBUG [PEWorker-2 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-24T08:29:39,776 INFO [PEWorker-2 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces 2024-11-24T08:29:39,797 DEBUG [PEWorker-2 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-24T08:29:39,798 DEBUG [PEWorker-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=30c28c82771d,40241,1732436977670, seqNum=-1] 2024-11-24T08:29:39,826 DEBUG [PEWorker-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-24T08:29:39,828 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-4-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57603, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-24T08:29:39,856 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 1.2300 sec 2024-11-24T08:29:39,857 INFO [master/30c28c82771d:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1732436979857, completionTime=-1 2024-11-24T08:29:39,859 INFO [master/30c28c82771d:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=3; waited=0ms, expected min=3 server(s), max=3 server(s), master is running 2024-11-24T08:29:39,859 DEBUG [master/30c28c82771d:0:becomeActiveMaster {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-24T08:29:39,917 INFO [master/30c28c82771d:0:becomeActiveMaster {}] assignment.AssignmentManager(1776): Number of RegionServers=3 2024-11-24T08:29:39,917 INFO [master/30c28c82771d:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732437039917 2024-11-24T08:29:39,917 INFO [master/30c28c82771d:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732437099917 2024-11-24T08:29:39,917 INFO [master/30c28c82771d:0:becomeActiveMaster {}] assignment.AssignmentManager(1783): Joined the cluster in 57 msec 2024-11-24T08:29:39,919 DEBUG [master/30c28c82771d:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 1588230740 changed from -1.0 to 0.0, refreshing cache 2024-11-24T08:29:39,931 INFO [master/30c28c82771d:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=30c28c82771d,37719,1732436976835-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-24T08:29:39,931 INFO [master/30c28c82771d:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=30c28c82771d,37719,1732436976835-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-24T08:29:39,931 INFO [master/30c28c82771d:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=30c28c82771d,37719,1732436976835-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-24T08:29:39,934 INFO [master/30c28c82771d:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-30c28c82771d:37719, period=300000, unit=MILLISECONDS is enabled. 2024-11-24T08:29:39,934 INFO [master/30c28c82771d:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2024-11-24T08:29:39,937 INFO [master/30c28c82771d:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled. 2024-11-24T08:29:39,942 DEBUG [master/30c28c82771d:0.Chore.1 {}] janitor.CatalogJanitor(180): 2024-11-24T08:29:39,971 INFO [master/30c28c82771d:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 2.161sec 2024-11-24T08:29:39,973 INFO [master/30c28c82771d:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2024-11-24T08:29:39,974 INFO [master/30c28c82771d:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2024-11-24T08:29:39,975 INFO [master/30c28c82771d:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2024-11-24T08:29:39,976 INFO [master/30c28c82771d:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2024-11-24T08:29:39,976 INFO [master/30c28c82771d:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2024-11-24T08:29:39,976 INFO [master/30c28c82771d:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=30c28c82771d,37719,1732436976835-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-24T08:29:39,977 INFO [master/30c28c82771d:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=30c28c82771d,37719,1732436976835-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2024-11-24T08:29:39,981 DEBUG [master/30c28c82771d:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds 2024-11-24T08:29:39,982 INFO [master/30c28c82771d:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2024-11-24T08:29:39,982 INFO [master/30c28c82771d:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=30c28c82771d,37719,1732436976835-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-24T08:29:40,062 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@38ad9a8d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T08:29:40,063 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 30c28c82771d,37719,-1 for getting cluster id 2024-11-24T08:29:40,066 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-24T08:29:40,077 DEBUG [HMaster-EventLoopGroup-2-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '8962100e-ff2b-4a0c-9fd7-8de6412af7e9' 2024-11-24T08:29:40,080 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-24T08:29:40,080 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "8962100e-ff2b-4a0c-9fd7-8de6412af7e9" 2024-11-24T08:29:40,080 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@46a739b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T08:29:40,080 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [30c28c82771d,37719,-1] 2024-11-24T08:29:40,084 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-24T08:29:40,086 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T08:29:40,086 INFO [HMaster-EventLoopGroup-2-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:44338, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-24T08:29:40,089 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@20190245, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T08:29:40,090 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-24T08:29:40,098 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=30c28c82771d,40241,1732436977670, seqNum=-1] 2024-11-24T08:29:40,099 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-24T08:29:40,101 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-4-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:55882, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-24T08:29:40,121 INFO [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=30c28c82771d,37719,1732436976835 2024-11-24T08:29:40,121 INFO [Time-limited test {}] wal.AbstractTestWALReplay(147): hbase.rootdir=hdfs://localhost:46165/hbase 2024-11-24T08:29:40,140 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestAsyncWALReplayCompressed#testNameConflictWhenSplit0 Thread=362, OpenFileDescriptor=607, MaxFileDescriptor=1048576, SystemLoadAverage=158, ProcessCount=11, AvailableMemoryMB=7140 2024-11-24T08:29:40,168 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-24T08:29:40,172 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-24T08:29:40,173 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-24T08:29:40,178 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-24266766, suffix=, logDir=hdfs://localhost:46165/hbase/WALs/hregion-24266766, archiveDir=hdfs://localhost:46165/hbase/oldWALs, maxLogs=32 2024-11-24T08:29:40,201 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-24266766/hregion-24266766.1732436980180, exclude list is [], retry=0 2024-11-24T08:29:40,206 DEBUG [AsyncFSWAL-8-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:34459,DS-6b4d1656-f49b-4a16-be49-2eb682c16e11,DISK] 2024-11-24T08:29:40,206 DEBUG [AsyncFSWAL-8-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33555,DS-8d97c3e2-1525-418c-a970-00a7157a0d31,DISK] 2024-11-24T08:29:40,207 DEBUG [AsyncFSWAL-8-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40533,DS-5eeb9666-6dd5-4a48-9fb1-3f9aff1acdca,DISK] 2024-11-24T08:29:40,212 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-24266766/hregion-24266766.1732436980180 2024-11-24T08:29:40,213 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:44131:44131),(127.0.0.1/127.0.0.1:39357:39357),(127.0.0.1/127.0.0.1:42673:42673)] 2024-11-24T08:29:40,213 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => 1a4ce1d57ea186e835c1cfcd56c3fe4f, NAME => 'testReplayEditsWrittenIntoWAL,,1732436980169.1a4ce1d57ea186e835c1cfcd56c3fe4f.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testReplayEditsWrittenIntoWAL', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:46165/hbase 2024-11-24T08:29:40,225 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33555 is added to blk_1073741838_1014 (size=64) 2024-11-24T08:29:40,226 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34459 is added to blk_1073741838_1014 (size=64) 2024-11-24T08:29:40,226 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40533 is added to blk_1073741838_1014 (size=64) 2024-11-24T08:29:40,227 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenIntoWAL,,1732436980169.1a4ce1d57ea186e835c1cfcd56c3fe4f.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T08:29:40,230 INFO [StoreOpener-1a4ce1d57ea186e835c1cfcd56c3fe4f-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 1a4ce1d57ea186e835c1cfcd56c3fe4f 2024-11-24T08:29:40,232 INFO [StoreOpener-1a4ce1d57ea186e835c1cfcd56c3fe4f-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1a4ce1d57ea186e835c1cfcd56c3fe4f columnFamilyName a 2024-11-24T08:29:40,233 DEBUG [StoreOpener-1a4ce1d57ea186e835c1cfcd56c3fe4f-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:29:40,233 INFO [StoreOpener-1a4ce1d57ea186e835c1cfcd56c3fe4f-1 {}] regionserver.HStore(327): Store=1a4ce1d57ea186e835c1cfcd56c3fe4f/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T08:29:40,234 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 1a4ce1d57ea186e835c1cfcd56c3fe4f 2024-11-24T08:29:40,235 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46165/hbase/data/default/testReplayEditsWrittenIntoWAL/1a4ce1d57ea186e835c1cfcd56c3fe4f 2024-11-24T08:29:40,235 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46165/hbase/data/default/testReplayEditsWrittenIntoWAL/1a4ce1d57ea186e835c1cfcd56c3fe4f 2024-11-24T08:29:40,236 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 1a4ce1d57ea186e835c1cfcd56c3fe4f 2024-11-24T08:29:40,236 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 1a4ce1d57ea186e835c1cfcd56c3fe4f 2024-11-24T08:29:40,241 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 1a4ce1d57ea186e835c1cfcd56c3fe4f 2024-11-24T08:29:40,245 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:46165/hbase/data/default/testReplayEditsWrittenIntoWAL/1a4ce1d57ea186e835c1cfcd56c3fe4f/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-24T08:29:40,246 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 1a4ce1d57ea186e835c1cfcd56c3fe4f; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=64133917, jitterRate=-0.0443301647901535}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-24T08:29:40,247 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 1a4ce1d57ea186e835c1cfcd56c3fe4f: Writing region info on filesystem at 1732436980228Initializing all the Stores at 1732436980230 (+2 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732436980230Cleaning up temporary data from old regions at 1732436980236 (+6 ms)Region opened successfully at 1732436980247 (+11 ms) 2024-11-24T08:29:40,247 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 1a4ce1d57ea186e835c1cfcd56c3fe4f, disabling compactions & flushes 2024-11-24T08:29:40,247 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsWrittenIntoWAL,,1732436980169.1a4ce1d57ea186e835c1cfcd56c3fe4f. 2024-11-24T08:29:40,247 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsWrittenIntoWAL,,1732436980169.1a4ce1d57ea186e835c1cfcd56c3fe4f. 2024-11-24T08:29:40,247 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsWrittenIntoWAL,,1732436980169.1a4ce1d57ea186e835c1cfcd56c3fe4f. after waiting 0 ms 2024-11-24T08:29:40,247 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsWrittenIntoWAL,,1732436980169.1a4ce1d57ea186e835c1cfcd56c3fe4f. 2024-11-24T08:29:40,247 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsWrittenIntoWAL,,1732436980169.1a4ce1d57ea186e835c1cfcd56c3fe4f. 2024-11-24T08:29:40,248 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 1a4ce1d57ea186e835c1cfcd56c3fe4f: Waiting for close lock at 1732436980247Disabling compacts and flushes for region at 1732436980247Disabling writes for close at 1732436980247Writing region close event to WAL at 1732436980247Closed at 1732436980247 2024-11-24T08:29:40,253 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40533 is added to blk_1073741837_1013 (size=93) 2024-11-24T08:29:40,253 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34459 is added to blk_1073741837_1013 (size=93) 2024-11-24T08:29:40,253 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33555 is added to blk_1073741837_1013 (size=93) 2024-11-24T08:29:40,261 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-11-24T08:29:40,261 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-24266766:(num 1732436980180) 2024-11-24T08:29:40,263 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor testMonitor 2024-11-24T08:29:40,279 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40533 is added to blk_1073741839_1015 (size=276) 2024-11-24T08:29:40,279 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33555 is added to blk_1073741839_1015 (size=276) 2024-11-24T08:29:40,280 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34459 is added to blk_1073741839_1015 (size=276) 2024-11-24T08:29:40,282 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor testMonitor 2024-11-24T08:29:40,296 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40533 is added to blk_1073741840_1016 (size=230) 2024-11-24T08:29:40,297 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34459 is added to blk_1073741840_1016 (size=230) 2024-11-24T08:29:40,298 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33555 is added to blk_1073741840_1016 (size=230) 2024-11-24T08:29:40,322 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:46165/hbase/WALs/testnameconflictwhensplit0-manual,16010,1732436980167/wal-1, size=276 (276bytes) 2024-11-24T08:29:40,323 DEBUG [Time-limited test {}] util.RecoverLeaseFSUtils(47): Initialize RecoverLeaseFSUtils 2024-11-24T08:29:40,323 DEBUG [Time-limited test {}] util.RecoverLeaseFSUtils(59): set recoverLeaseMethod to org.apache.hadoop.fs.LeaseRecoverable.recoverLease() 2024-11-24T08:29:40,323 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:46165/hbase/WALs/testnameconflictwhensplit0-manual,16010,1732436980167/wal-1 2024-11-24T08:29:40,329 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:46165/hbase/WALs/testnameconflictwhensplit0-manual,16010,1732436980167/wal-1 after 4ms 2024-11-24T08:29:40,335 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:46165/hbase/WALs/testnameconflictwhensplit0-manual,16010,1732436980167/wal-1: isRecoveredEdits=false, hasTagCompression=true, hasValueCompression=false, valueCompressionType=null 2024-11-24T08:29:40,336 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:46165/hbase/WALs/testnameconflictwhensplit0-manual,16010,1732436980167/wal-1 took 15ms 2024-11-24T08:29:40,340 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:46165/hbase/WALs/testnameconflictwhensplit0-manual,16010,1732436980167/wal-1 so closing down 2024-11-24T08:29:40,343 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000001-wal-1.temp 2024-11-24T08:29:40,345 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-24T08:29:40,346 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:46165/hbase/data/default/testReplayEditsWrittenIntoWAL/1a4ce1d57ea186e835c1cfcd56c3fe4f/recovered.edits/0000000000000000001-wal-1.temp 2024-11-24T08:29:40,347 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-11-24T08:29:40,359 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33555 is added to blk_1073741841_1017 (size=276) 2024-11-24T08:29:40,359 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40533 is added to blk_1073741841_1017 (size=276) 2024-11-24T08:29:40,359 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34459 is added to blk_1073741841_1017 (size=276) 2024-11-24T08:29:40,365 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:46165/hbase/data/default/testReplayEditsWrittenIntoWAL/1a4ce1d57ea186e835c1cfcd56c3fe4f/recovered.edits/0000000000000000001-wal-1.temp (wrote 2 edits, skipped 0 edits in 0 ms) 2024-11-24T08:29:40,368 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:46165/hbase/data/default/testReplayEditsWrittenIntoWAL/1a4ce1d57ea186e835c1cfcd56c3fe4f/recovered.edits/0000000000000000001-wal-1.temp to hdfs://localhost:46165/hbase/data/default/testReplayEditsWrittenIntoWAL/1a4ce1d57ea186e835c1cfcd56c3fe4f/recovered.edits/0000000000000000002 2024-11-24T08:29:40,371 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 2 edits across 1 Regions in 31 ms; skipped=0; WAL=hdfs://localhost:46165/hbase/WALs/testnameconflictwhensplit0-manual,16010,1732436980167/wal-1, size=276, length=276, corrupted=false, cancelled=false 2024-11-24T08:29:40,371 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:46165/hbase/WALs/testnameconflictwhensplit0-manual,16010,1732436980167/wal-1, journal: Splitting hdfs://localhost:46165/hbase/WALs/testnameconflictwhensplit0-manual,16010,1732436980167/wal-1, size=276 (276bytes) at 1732436980322Finishing writing output for hdfs://localhost:46165/hbase/WALs/testnameconflictwhensplit0-manual,16010,1732436980167/wal-1 so closing down at 1732436980340 (+18 ms)Creating recovered edits writer path=hdfs://localhost:46165/hbase/data/default/testReplayEditsWrittenIntoWAL/1a4ce1d57ea186e835c1cfcd56c3fe4f/recovered.edits/0000000000000000001-wal-1.temp at 1732436980346 (+6 ms)3 split writer threads finished at 1732436980347 (+1 ms)Closed recovered edits writer path=hdfs://localhost:46165/hbase/data/default/testReplayEditsWrittenIntoWAL/1a4ce1d57ea186e835c1cfcd56c3fe4f/recovered.edits/0000000000000000001-wal-1.temp (wrote 2 edits, skipped 0 edits in 0 ms) at 1732436980365 (+18 ms)Rename recovered edits hdfs://localhost:46165/hbase/data/default/testReplayEditsWrittenIntoWAL/1a4ce1d57ea186e835c1cfcd56c3fe4f/recovered.edits/0000000000000000001-wal-1.temp to hdfs://localhost:46165/hbase/data/default/testReplayEditsWrittenIntoWAL/1a4ce1d57ea186e835c1cfcd56c3fe4f/recovered.edits/0000000000000000002 at 1732436980368 (+3 ms)Processed 2 edits across 1 Regions in 31 ms; skipped=0; WAL=hdfs://localhost:46165/hbase/WALs/testnameconflictwhensplit0-manual,16010,1732436980167/wal-1, size=276, length=276, corrupted=false, cancelled=false at 1732436980371 (+3 ms) 2024-11-24T08:29:40,387 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:46165/hbase/WALs/testnameconflictwhensplit0-manual,16010,1732436980167/wal-2, size=230 (230bytes) 2024-11-24T08:29:40,387 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:46165/hbase/WALs/testnameconflictwhensplit0-manual,16010,1732436980167/wal-2 2024-11-24T08:29:40,388 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:46165/hbase/WALs/testnameconflictwhensplit0-manual,16010,1732436980167/wal-2 after 1ms 2024-11-24T08:29:40,392 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:46165/hbase/WALs/testnameconflictwhensplit0-manual,16010,1732436980167/wal-2: isRecoveredEdits=false, hasTagCompression=true, hasValueCompression=false, valueCompressionType=null 2024-11-24T08:29:40,393 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:46165/hbase/WALs/testnameconflictwhensplit0-manual,16010,1732436980167/wal-2 took 6ms 2024-11-24T08:29:40,395 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:46165/hbase/WALs/testnameconflictwhensplit0-manual,16010,1732436980167/wal-2 so closing down 2024-11-24T08:29:40,395 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-24T08:29:40,397 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000002-wal-2.temp 2024-11-24T08:29:40,399 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:46165/hbase/data/default/testReplayEditsWrittenIntoWAL/1a4ce1d57ea186e835c1cfcd56c3fe4f/recovered.edits/0000000000000000002-wal-2.temp 2024-11-24T08:29:40,399 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-11-24T08:29:40,407 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33555 is added to blk_1073741842_1018 (size=230) 2024-11-24T08:29:40,407 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34459 is added to blk_1073741842_1018 (size=230) 2024-11-24T08:29:40,408 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40533 is added to blk_1073741842_1018 (size=230) 2024-11-24T08:29:40,409 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:46165/hbase/data/default/testReplayEditsWrittenIntoWAL/1a4ce1d57ea186e835c1cfcd56c3fe4f/recovered.edits/0000000000000000002-wal-2.temp (wrote 1 edits, skipped 0 edits in 0 ms) 2024-11-24T08:29:40,413 DEBUG [split-log-closeStream-pool-0 {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:46165/hbase/data/default/testReplayEditsWrittenIntoWAL/1a4ce1d57ea186e835c1cfcd56c3fe4f/recovered.edits/0000000000000000002: isRecoveredEdits=true, hasTagCompression=true, hasValueCompression=false, valueCompressionType=null 2024-11-24T08:29:40,416 WARN [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(184): Found existing old edits file and we have less entries. Deleting hdfs://localhost:46165/hbase/data/default/testReplayEditsWrittenIntoWAL/1a4ce1d57ea186e835c1cfcd56c3fe4f/recovered.edits/0000000000000000002-wal-2.temp, length=230 2024-11-24T08:29:40,417 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 1 edits across 1 Regions in 24 ms; skipped=0; WAL=hdfs://localhost:46165/hbase/WALs/testnameconflictwhensplit0-manual,16010,1732436980167/wal-2, size=230, length=230, corrupted=false, cancelled=false 2024-11-24T08:29:40,418 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:46165/hbase/WALs/testnameconflictwhensplit0-manual,16010,1732436980167/wal-2, journal: Splitting hdfs://localhost:46165/hbase/WALs/testnameconflictwhensplit0-manual,16010,1732436980167/wal-2, size=230 (230bytes) at 1732436980387Finishing writing output for hdfs://localhost:46165/hbase/WALs/testnameconflictwhensplit0-manual,16010,1732436980167/wal-2 so closing down at 1732436980395 (+8 ms)Creating recovered edits writer path=hdfs://localhost:46165/hbase/data/default/testReplayEditsWrittenIntoWAL/1a4ce1d57ea186e835c1cfcd56c3fe4f/recovered.edits/0000000000000000002-wal-2.temp at 1732436980399 (+4 ms)3 split writer threads finished at 1732436980399Closed recovered edits writer path=hdfs://localhost:46165/hbase/data/default/testReplayEditsWrittenIntoWAL/1a4ce1d57ea186e835c1cfcd56c3fe4f/recovered.edits/0000000000000000002-wal-2.temp (wrote 1 edits, skipped 0 edits in 0 ms) at 1732436980409 (+10 ms)Processed 1 edits across 1 Regions in 24 ms; skipped=0; WAL=hdfs://localhost:46165/hbase/WALs/testnameconflictwhensplit0-manual,16010,1732436980167/wal-2, size=230, length=230, corrupted=false, cancelled=false at 1732436980417 (+8 ms) 2024-11-24T08:29:40,418 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-24T08:29:40,420 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:46165/hbase/WALs/testnameconflictwhensplit0-manual,16010,1732436980167, archiveDir=hdfs://localhost:46165/hbase/oldWALs, maxLogs=32 2024-11-24T08:29:40,435 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testnameconflictwhensplit0-manual,16010,1732436980167/wal.1732436980421, exclude list is [], retry=0 2024-11-24T08:29:40,439 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:34459,DS-6b4d1656-f49b-4a16-be49-2eb682c16e11,DISK] 2024-11-24T08:29:40,440 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40533,DS-5eeb9666-6dd5-4a48-9fb1-3f9aff1acdca,DISK] 2024-11-24T08:29:40,440 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33555,DS-8d97c3e2-1525-418c-a970-00a7157a0d31,DISK] 2024-11-24T08:29:40,443 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testnameconflictwhensplit0-manual,16010,1732436980167/wal.1732436980421 2024-11-24T08:29:40,443 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:44131:44131),(127.0.0.1/127.0.0.1:42673:42673),(127.0.0.1/127.0.0.1:39357:39357)] 2024-11-24T08:29:40,443 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1a4ce1d57ea186e835c1cfcd56c3fe4f, NAME => 'testReplayEditsWrittenIntoWAL,,1732436980169.1a4ce1d57ea186e835c1cfcd56c3fe4f.', STARTKEY => '', ENDKEY => ''} 2024-11-24T08:29:40,443 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenIntoWAL,,1732436980169.1a4ce1d57ea186e835c1cfcd56c3fe4f.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T08:29:40,443 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 1a4ce1d57ea186e835c1cfcd56c3fe4f 2024-11-24T08:29:40,443 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 1a4ce1d57ea186e835c1cfcd56c3fe4f 2024-11-24T08:29:40,446 INFO [StoreOpener-1a4ce1d57ea186e835c1cfcd56c3fe4f-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 1a4ce1d57ea186e835c1cfcd56c3fe4f 2024-11-24T08:29:40,448 INFO [StoreOpener-1a4ce1d57ea186e835c1cfcd56c3fe4f-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1a4ce1d57ea186e835c1cfcd56c3fe4f columnFamilyName a 2024-11-24T08:29:40,448 DEBUG [StoreOpener-1a4ce1d57ea186e835c1cfcd56c3fe4f-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:29:40,449 INFO [StoreOpener-1a4ce1d57ea186e835c1cfcd56c3fe4f-1 {}] regionserver.HStore(327): Store=1a4ce1d57ea186e835c1cfcd56c3fe4f/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T08:29:40,450 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 1a4ce1d57ea186e835c1cfcd56c3fe4f 2024-11-24T08:29:40,451 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46165/hbase/data/default/testReplayEditsWrittenIntoWAL/1a4ce1d57ea186e835c1cfcd56c3fe4f 2024-11-24T08:29:40,455 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:46165/hbase/data/default/testReplayEditsWrittenIntoWAL/1a4ce1d57ea186e835c1cfcd56c3fe4f 2024-11-24T08:29:40,457 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:46165/hbase/data/default/testReplayEditsWrittenIntoWAL/1a4ce1d57ea186e835c1cfcd56c3fe4f/recovered.edits/0000000000000000002 2024-11-24T08:29:40,461 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:46165/hbase/data/default/testReplayEditsWrittenIntoWAL/1a4ce1d57ea186e835c1cfcd56c3fe4f/recovered.edits/0000000000000000002: isRecoveredEdits=true, hasTagCompression=true, hasValueCompression=false, valueCompressionType=null 2024-11-24T08:29:40,469 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 2, skipped 0, firstSequenceIdInLog=1, maxSequenceIdInLog=2, path=hdfs://localhost:46165/hbase/data/default/testReplayEditsWrittenIntoWAL/1a4ce1d57ea186e835c1cfcd56c3fe4f/recovered.edits/0000000000000000002 2024-11-24T08:29:40,473 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 1a4ce1d57ea186e835c1cfcd56c3fe4f 1/1 column families, dataSize=108 B heapSize=512 B 2024-11-24T08:29:40,541 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46165/hbase/data/default/testReplayEditsWrittenIntoWAL/1a4ce1d57ea186e835c1cfcd56c3fe4f/.tmp/a/87aa0da6ef36404caa086440a2cfb79b is 58, key is testReplayEditsWrittenIntoWAL/a:1/1732436980261/Put/seqid=0 2024-11-24T08:29:40,558 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40533 is added to blk_1073741844_1020 (size=5170) 2024-11-24T08:29:40,559 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33555 is added to blk_1073741844_1020 (size=5170) 2024-11-24T08:29:40,559 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34459 is added to blk_1073741844_1020 (size=5170) 2024-11-24T08:29:40,560 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=108 B at sequenceid=2 (bloomFilter=true), to=hdfs://localhost:46165/hbase/data/default/testReplayEditsWrittenIntoWAL/1a4ce1d57ea186e835c1cfcd56c3fe4f/.tmp/a/87aa0da6ef36404caa086440a2cfb79b 2024-11-24T08:29:40,604 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46165/hbase/data/default/testReplayEditsWrittenIntoWAL/1a4ce1d57ea186e835c1cfcd56c3fe4f/.tmp/a/87aa0da6ef36404caa086440a2cfb79b as hdfs://localhost:46165/hbase/data/default/testReplayEditsWrittenIntoWAL/1a4ce1d57ea186e835c1cfcd56c3fe4f/a/87aa0da6ef36404caa086440a2cfb79b 2024-11-24T08:29:40,615 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46165/hbase/data/default/testReplayEditsWrittenIntoWAL/1a4ce1d57ea186e835c1cfcd56c3fe4f/a/87aa0da6ef36404caa086440a2cfb79b, entries=2, sequenceid=2, filesize=5.0 K 2024-11-24T08:29:40,620 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~108 B/108, heapSize ~496 B/496, currentSize=0 B/0 for 1a4ce1d57ea186e835c1cfcd56c3fe4f in 148ms, sequenceid=2, compaction requested=false; wal=null 2024-11-24T08:29:40,622 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:46165/hbase/data/default/testReplayEditsWrittenIntoWAL/1a4ce1d57ea186e835c1cfcd56c3fe4f/recovered.edits/0000000000000000002 2024-11-24T08:29:40,623 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 1a4ce1d57ea186e835c1cfcd56c3fe4f 2024-11-24T08:29:40,623 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 1a4ce1d57ea186e835c1cfcd56c3fe4f 2024-11-24T08:29:40,627 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 1a4ce1d57ea186e835c1cfcd56c3fe4f 2024-11-24T08:29:40,630 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:46165/hbase/data/default/testReplayEditsWrittenIntoWAL/1a4ce1d57ea186e835c1cfcd56c3fe4f/recovered.edits/2.seqid, newMaxSeqId=2, maxSeqId=1 2024-11-24T08:29:40,632 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 1a4ce1d57ea186e835c1cfcd56c3fe4f; next sequenceid=3; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=60086544, jitterRate=-0.10464072227478027}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-24T08:29:40,632 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 1a4ce1d57ea186e835c1cfcd56c3fe4f: Writing region info on filesystem at 1732436980444Initializing all the Stores at 1732436980446 (+2 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732436980446Obtaining lock to block concurrent updates at 1732436980473 (+27 ms)Preparing flush snapshotting stores in 1a4ce1d57ea186e835c1cfcd56c3fe4f at 1732436980473Finished memstore snapshotting testReplayEditsWrittenIntoWAL,,1732436980169.1a4ce1d57ea186e835c1cfcd56c3fe4f., syncing WAL and waiting on mvcc, flushsize=dataSize=108, getHeapSize=496, getOffHeapSize=0, getCellsCount=2 at 1732436980477 (+4 ms)Flushing stores of testReplayEditsWrittenIntoWAL,,1732436980169.1a4ce1d57ea186e835c1cfcd56c3fe4f. at 1732436980477Flushing 1a4ce1d57ea186e835c1cfcd56c3fe4f/a: creating writer at 1732436980479 (+2 ms)Flushing 1a4ce1d57ea186e835c1cfcd56c3fe4f/a: appending metadata at 1732436980529 (+50 ms)Flushing 1a4ce1d57ea186e835c1cfcd56c3fe4f/a: closing flushed file at 1732436980533 (+4 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@32868b15: reopening flushed file at 1732436980602 (+69 ms)Finished flush of dataSize ~108 B/108, heapSize ~496 B/496, currentSize=0 B/0 for 1a4ce1d57ea186e835c1cfcd56c3fe4f in 148ms, sequenceid=2, compaction requested=false; wal=null at 1732436980620 (+18 ms)Cleaning up temporary data from old regions at 1732436980623 (+3 ms)Region opened successfully at 1732436980632 (+9 ms) 2024-11-24T08:29:40,663 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestAsyncWALReplayCompressed#testNameConflictWhenSplit0 Thread=374 (was 362) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1541013681_22 at /127.0.0.1:35512 [Receiving block BP-1907186785-172.17.0.2-1732436973683:blk_1073741843_1019] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1541013681_22 at /127.0.0.1:54076 [Receiving block BP-1907186785-172.17.0.2-1732436973683:blk_1073741843_1019] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-1907186785-172.17.0.2-1732436973683:blk_1073741843_1019, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-1907186785-172.17.0.2-1732436973683:blk_1073741843_1019, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-0-hdfs://localhost:46165/hbase-prefix:default java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-1907186785-172.17.0.2-1732436973683:blk_1073741843_1019, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-8-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1541013681_22 at /127.0.0.1:43204 [Receiving block BP-1907186785-172.17.0.2-1732436973683:blk_1073741843_1019] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: TestAsyncWALReplay-pool-0 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:62) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:883) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1541013681_22 at /127.0.0.1:54068 [Waiting for operation #8] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-8-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-8-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1541013681_22 at /127.0.0.1:43094 [Waiting for operation #10] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=703 (was 607) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=158 (was 158), ProcessCount=11 (was 11), AvailableMemoryMB=7125 (was 7140) 2024-11-24T08:29:40,676 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestAsyncWALReplayCompressed#testNameConflictWhenSplit1 Thread=374, OpenFileDescriptor=703, MaxFileDescriptor=1048576, SystemLoadAverage=158, ProcessCount=11, AvailableMemoryMB=7124 2024-11-24T08:29:40,692 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-24T08:29:40,694 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-24T08:29:40,695 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-24T08:29:40,698 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-94675450, suffix=, logDir=hdfs://localhost:46165/hbase/WALs/hregion-94675450, archiveDir=hdfs://localhost:46165/hbase/oldWALs, maxLogs=32 2024-11-24T08:29:40,712 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-94675450/hregion-94675450.1732436980699, exclude list is [], retry=0 2024-11-24T08:29:40,716 DEBUG [AsyncFSWAL-10-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40533,DS-5eeb9666-6dd5-4a48-9fb1-3f9aff1acdca,DISK] 2024-11-24T08:29:40,717 DEBUG [AsyncFSWAL-10-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:34459,DS-6b4d1656-f49b-4a16-be49-2eb682c16e11,DISK] 2024-11-24T08:29:40,718 DEBUG [AsyncFSWAL-10-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33555,DS-8d97c3e2-1525-418c-a970-00a7157a0d31,DISK] 2024-11-24T08:29:40,725 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-94675450/hregion-94675450.1732436980699 2024-11-24T08:29:40,726 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:42673:42673),(127.0.0.1/127.0.0.1:44131:44131),(127.0.0.1/127.0.0.1:39357:39357)] 2024-11-24T08:29:40,726 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => 40f4f676921d6bc5a898ddcaf9f09897, NAME => 'testReplayEditsWrittenIntoWAL,,1732436980693.40f4f676921d6bc5a898ddcaf9f09897.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testReplayEditsWrittenIntoWAL', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:46165/hbase 2024-11-24T08:29:40,737 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34459 is added to blk_1073741846_1022 (size=64) 2024-11-24T08:29:40,738 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33555 is added to blk_1073741846_1022 (size=64) 2024-11-24T08:29:40,738 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40533 is added to blk_1073741846_1022 (size=64) 2024-11-24T08:29:40,740 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenIntoWAL,,1732436980693.40f4f676921d6bc5a898ddcaf9f09897.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T08:29:40,742 INFO [StoreOpener-40f4f676921d6bc5a898ddcaf9f09897-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 40f4f676921d6bc5a898ddcaf9f09897 2024-11-24T08:29:40,745 INFO [StoreOpener-40f4f676921d6bc5a898ddcaf9f09897-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 40f4f676921d6bc5a898ddcaf9f09897 columnFamilyName a 2024-11-24T08:29:40,745 DEBUG [StoreOpener-40f4f676921d6bc5a898ddcaf9f09897-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:29:40,745 INFO [StoreOpener-40f4f676921d6bc5a898ddcaf9f09897-1 {}] regionserver.HStore(327): Store=40f4f676921d6bc5a898ddcaf9f09897/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T08:29:40,746 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 40f4f676921d6bc5a898ddcaf9f09897 2024-11-24T08:29:40,747 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46165/hbase/data/default/testReplayEditsWrittenIntoWAL/40f4f676921d6bc5a898ddcaf9f09897 2024-11-24T08:29:40,748 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46165/hbase/data/default/testReplayEditsWrittenIntoWAL/40f4f676921d6bc5a898ddcaf9f09897 2024-11-24T08:29:40,748 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 40f4f676921d6bc5a898ddcaf9f09897 2024-11-24T08:29:40,748 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 40f4f676921d6bc5a898ddcaf9f09897 2024-11-24T08:29:40,751 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 40f4f676921d6bc5a898ddcaf9f09897 2024-11-24T08:29:40,754 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:46165/hbase/data/default/testReplayEditsWrittenIntoWAL/40f4f676921d6bc5a898ddcaf9f09897/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-24T08:29:40,755 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 40f4f676921d6bc5a898ddcaf9f09897; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=73174655, jitterRate=0.09038732945919037}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-24T08:29:40,755 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 40f4f676921d6bc5a898ddcaf9f09897: Writing region info on filesystem at 1732436980740Initializing all the Stores at 1732436980741 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732436980741Cleaning up temporary data from old regions at 1732436980748 (+7 ms)Region opened successfully at 1732436980755 (+7 ms) 2024-11-24T08:29:40,756 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 40f4f676921d6bc5a898ddcaf9f09897, disabling compactions & flushes 2024-11-24T08:29:40,756 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsWrittenIntoWAL,,1732436980693.40f4f676921d6bc5a898ddcaf9f09897. 2024-11-24T08:29:40,756 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsWrittenIntoWAL,,1732436980693.40f4f676921d6bc5a898ddcaf9f09897. 2024-11-24T08:29:40,756 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsWrittenIntoWAL,,1732436980693.40f4f676921d6bc5a898ddcaf9f09897. after waiting 0 ms 2024-11-24T08:29:40,756 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsWrittenIntoWAL,,1732436980693.40f4f676921d6bc5a898ddcaf9f09897. 2024-11-24T08:29:40,757 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsWrittenIntoWAL,,1732436980693.40f4f676921d6bc5a898ddcaf9f09897. 2024-11-24T08:29:40,757 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 40f4f676921d6bc5a898ddcaf9f09897: Waiting for close lock at 1732436980756Disabling compacts and flushes for region at 1732436980756Disabling writes for close at 1732436980756Writing region close event to WAL at 1732436980756Closed at 1732436980756 2024-11-24T08:29:40,761 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40533 is added to blk_1073741845_1021 (size=93) 2024-11-24T08:29:40,761 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33555 is added to blk_1073741845_1021 (size=93) 2024-11-24T08:29:40,762 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34459 is added to blk_1073741845_1021 (size=93) 2024-11-24T08:29:40,765 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-11-24T08:29:40,765 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-94675450:(num 1732436980699) 2024-11-24T08:29:40,766 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor testMonitor 2024-11-24T08:29:40,780 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40533 is added to blk_1073741847_1023 (size=276) 2024-11-24T08:29:40,780 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33555 is added to blk_1073741847_1023 (size=276) 2024-11-24T08:29:40,781 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34459 is added to blk_1073741847_1023 (size=276) 2024-11-24T08:29:40,782 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor testMonitor 2024-11-24T08:29:40,793 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33555 is added to blk_1073741848_1024 (size=230) 2024-11-24T08:29:40,794 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34459 is added to blk_1073741848_1024 (size=230) 2024-11-24T08:29:40,795 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40533 is added to blk_1073741848_1024 (size=230) 2024-11-24T08:29:40,817 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:46165/hbase/WALs/testnameconflictwhensplit1-manual,16010,1732436980691/wal-2, size=230 (230bytes) 2024-11-24T08:29:40,817 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:46165/hbase/WALs/testnameconflictwhensplit1-manual,16010,1732436980691/wal-2 2024-11-24T08:29:40,818 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:46165/hbase/WALs/testnameconflictwhensplit1-manual,16010,1732436980691/wal-2 after 1ms 2024-11-24T08:29:40,823 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:46165/hbase/WALs/testnameconflictwhensplit1-manual,16010,1732436980691/wal-2: isRecoveredEdits=false, hasTagCompression=true, hasValueCompression=false, valueCompressionType=null 2024-11-24T08:29:40,823 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:46165/hbase/WALs/testnameconflictwhensplit1-manual,16010,1732436980691/wal-2 took 6ms 2024-11-24T08:29:40,826 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:46165/hbase/WALs/testnameconflictwhensplit1-manual,16010,1732436980691/wal-2 so closing down 2024-11-24T08:29:40,826 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-24T08:29:40,829 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000002-wal-2.temp 2024-11-24T08:29:40,836 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:46165/hbase/data/default/testReplayEditsWrittenIntoWAL/40f4f676921d6bc5a898ddcaf9f09897/recovered.edits/0000000000000000002-wal-2.temp 2024-11-24T08:29:40,836 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-11-24T08:29:40,847 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33555 is added to blk_1073741849_1025 (size=230) 2024-11-24T08:29:40,847 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40533 is added to blk_1073741849_1025 (size=230) 2024-11-24T08:29:40,848 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34459 is added to blk_1073741849_1025 (size=230) 2024-11-24T08:29:40,849 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:46165/hbase/data/default/testReplayEditsWrittenIntoWAL/40f4f676921d6bc5a898ddcaf9f09897/recovered.edits/0000000000000000002-wal-2.temp (wrote 1 edits, skipped 0 edits in 0 ms) 2024-11-24T08:29:40,851 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:46165/hbase/data/default/testReplayEditsWrittenIntoWAL/40f4f676921d6bc5a898ddcaf9f09897/recovered.edits/0000000000000000002-wal-2.temp to hdfs://localhost:46165/hbase/data/default/testReplayEditsWrittenIntoWAL/40f4f676921d6bc5a898ddcaf9f09897/recovered.edits/0000000000000000002 2024-11-24T08:29:40,851 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 1 edits across 1 Regions in 28 ms; skipped=0; WAL=hdfs://localhost:46165/hbase/WALs/testnameconflictwhensplit1-manual,16010,1732436980691/wal-2, size=230, length=230, corrupted=false, cancelled=false 2024-11-24T08:29:40,851 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:46165/hbase/WALs/testnameconflictwhensplit1-manual,16010,1732436980691/wal-2, journal: Splitting hdfs://localhost:46165/hbase/WALs/testnameconflictwhensplit1-manual,16010,1732436980691/wal-2, size=230 (230bytes) at 1732436980817Finishing writing output for hdfs://localhost:46165/hbase/WALs/testnameconflictwhensplit1-manual,16010,1732436980691/wal-2 so closing down at 1732436980826 (+9 ms)Creating recovered edits writer path=hdfs://localhost:46165/hbase/data/default/testReplayEditsWrittenIntoWAL/40f4f676921d6bc5a898ddcaf9f09897/recovered.edits/0000000000000000002-wal-2.temp at 1732436980836 (+10 ms)3 split writer threads finished at 1732436980836Closed recovered edits writer path=hdfs://localhost:46165/hbase/data/default/testReplayEditsWrittenIntoWAL/40f4f676921d6bc5a898ddcaf9f09897/recovered.edits/0000000000000000002-wal-2.temp (wrote 1 edits, skipped 0 edits in 0 ms) at 1732436980849 (+13 ms)Rename recovered edits hdfs://localhost:46165/hbase/data/default/testReplayEditsWrittenIntoWAL/40f4f676921d6bc5a898ddcaf9f09897/recovered.edits/0000000000000000002-wal-2.temp to hdfs://localhost:46165/hbase/data/default/testReplayEditsWrittenIntoWAL/40f4f676921d6bc5a898ddcaf9f09897/recovered.edits/0000000000000000002 at 1732436980851 (+2 ms)Processed 1 edits across 1 Regions in 28 ms; skipped=0; WAL=hdfs://localhost:46165/hbase/WALs/testnameconflictwhensplit1-manual,16010,1732436980691/wal-2, size=230, length=230, corrupted=false, cancelled=false at 1732436980851 2024-11-24T08:29:40,868 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:46165/hbase/WALs/testnameconflictwhensplit1-manual,16010,1732436980691/wal-1, size=276 (276bytes) 2024-11-24T08:29:40,868 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:46165/hbase/WALs/testnameconflictwhensplit1-manual,16010,1732436980691/wal-1 2024-11-24T08:29:40,869 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:46165/hbase/WALs/testnameconflictwhensplit1-manual,16010,1732436980691/wal-1 after 0ms 2024-11-24T08:29:40,873 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:46165/hbase/WALs/testnameconflictwhensplit1-manual,16010,1732436980691/wal-1: isRecoveredEdits=false, hasTagCompression=true, hasValueCompression=false, valueCompressionType=null 2024-11-24T08:29:40,873 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:46165/hbase/WALs/testnameconflictwhensplit1-manual,16010,1732436980691/wal-1 took 6ms 2024-11-24T08:29:40,875 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:46165/hbase/WALs/testnameconflictwhensplit1-manual,16010,1732436980691/wal-1 so closing down 2024-11-24T08:29:40,876 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-24T08:29:40,878 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000001-wal-1.temp 2024-11-24T08:29:40,879 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:46165/hbase/data/default/testReplayEditsWrittenIntoWAL/40f4f676921d6bc5a898ddcaf9f09897/recovered.edits/0000000000000000001-wal-1.temp 2024-11-24T08:29:40,880 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-11-24T08:29:40,887 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33555 is added to blk_1073741850_1026 (size=276) 2024-11-24T08:29:40,887 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40533 is added to blk_1073741850_1026 (size=276) 2024-11-24T08:29:40,888 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34459 is added to blk_1073741850_1026 (size=276) 2024-11-24T08:29:40,889 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:46165/hbase/data/default/testReplayEditsWrittenIntoWAL/40f4f676921d6bc5a898ddcaf9f09897/recovered.edits/0000000000000000001-wal-1.temp (wrote 2 edits, skipped 0 edits in 0 ms) 2024-11-24T08:29:40,893 DEBUG [split-log-closeStream-pool-0 {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:46165/hbase/data/default/testReplayEditsWrittenIntoWAL/40f4f676921d6bc5a898ddcaf9f09897/recovered.edits/0000000000000000002: isRecoveredEdits=true, hasTagCompression=true, hasValueCompression=false, valueCompressionType=null 2024-11-24T08:29:40,895 WARN [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(175): Found existing old edits file. It could be the result of a previous failed split attempt or we have duplicated wal entries. Deleting hdfs://localhost:46165/hbase/data/default/testReplayEditsWrittenIntoWAL/40f4f676921d6bc5a898ddcaf9f09897/recovered.edits/0000000000000000002, length=230 2024-11-24T08:29:40,898 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:46165/hbase/data/default/testReplayEditsWrittenIntoWAL/40f4f676921d6bc5a898ddcaf9f09897/recovered.edits/0000000000000000001-wal-1.temp to hdfs://localhost:46165/hbase/data/default/testReplayEditsWrittenIntoWAL/40f4f676921d6bc5a898ddcaf9f09897/recovered.edits/0000000000000000002 2024-11-24T08:29:40,898 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 2 edits across 1 Regions in 25 ms; skipped=0; WAL=hdfs://localhost:46165/hbase/WALs/testnameconflictwhensplit1-manual,16010,1732436980691/wal-1, size=276, length=276, corrupted=false, cancelled=false 2024-11-24T08:29:40,898 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:46165/hbase/WALs/testnameconflictwhensplit1-manual,16010,1732436980691/wal-1, journal: Splitting hdfs://localhost:46165/hbase/WALs/testnameconflictwhensplit1-manual,16010,1732436980691/wal-1, size=276 (276bytes) at 1732436980868Finishing writing output for hdfs://localhost:46165/hbase/WALs/testnameconflictwhensplit1-manual,16010,1732436980691/wal-1 so closing down at 1732436980876 (+8 ms)Creating recovered edits writer path=hdfs://localhost:46165/hbase/data/default/testReplayEditsWrittenIntoWAL/40f4f676921d6bc5a898ddcaf9f09897/recovered.edits/0000000000000000001-wal-1.temp at 1732436980879 (+3 ms)3 split writer threads finished at 1732436980880 (+1 ms)Closed recovered edits writer path=hdfs://localhost:46165/hbase/data/default/testReplayEditsWrittenIntoWAL/40f4f676921d6bc5a898ddcaf9f09897/recovered.edits/0000000000000000001-wal-1.temp (wrote 2 edits, skipped 0 edits in 0 ms) at 1732436980889 (+9 ms)Rename recovered edits hdfs://localhost:46165/hbase/data/default/testReplayEditsWrittenIntoWAL/40f4f676921d6bc5a898ddcaf9f09897/recovered.edits/0000000000000000001-wal-1.temp to hdfs://localhost:46165/hbase/data/default/testReplayEditsWrittenIntoWAL/40f4f676921d6bc5a898ddcaf9f09897/recovered.edits/0000000000000000002 at 1732436980898 (+9 ms)Processed 2 edits across 1 Regions in 25 ms; skipped=0; WAL=hdfs://localhost:46165/hbase/WALs/testnameconflictwhensplit1-manual,16010,1732436980691/wal-1, size=276, length=276, corrupted=false, cancelled=false at 1732436980898 2024-11-24T08:29:40,899 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-24T08:29:40,901 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:46165/hbase/WALs/testnameconflictwhensplit1-manual,16010,1732436980691, archiveDir=hdfs://localhost:46165/hbase/oldWALs, maxLogs=32 2024-11-24T08:29:40,916 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testnameconflictwhensplit1-manual,16010,1732436980691/wal.1732436980902, exclude list is [], retry=0 2024-11-24T08:29:40,920 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40533,DS-5eeb9666-6dd5-4a48-9fb1-3f9aff1acdca,DISK] 2024-11-24T08:29:40,920 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:34459,DS-6b4d1656-f49b-4a16-be49-2eb682c16e11,DISK] 2024-11-24T08:29:40,921 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33555,DS-8d97c3e2-1525-418c-a970-00a7157a0d31,DISK] 2024-11-24T08:29:40,924 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testnameconflictwhensplit1-manual,16010,1732436980691/wal.1732436980902 2024-11-24T08:29:40,924 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:42673:42673),(127.0.0.1/127.0.0.1:44131:44131),(127.0.0.1/127.0.0.1:39357:39357)] 2024-11-24T08:29:40,924 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 40f4f676921d6bc5a898ddcaf9f09897, NAME => 'testReplayEditsWrittenIntoWAL,,1732436980693.40f4f676921d6bc5a898ddcaf9f09897.', STARTKEY => '', ENDKEY => ''} 2024-11-24T08:29:40,924 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenIntoWAL,,1732436980693.40f4f676921d6bc5a898ddcaf9f09897.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T08:29:40,925 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 40f4f676921d6bc5a898ddcaf9f09897 2024-11-24T08:29:40,925 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 40f4f676921d6bc5a898ddcaf9f09897 2024-11-24T08:29:40,927 INFO [StoreOpener-40f4f676921d6bc5a898ddcaf9f09897-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 40f4f676921d6bc5a898ddcaf9f09897 2024-11-24T08:29:40,929 INFO [StoreOpener-40f4f676921d6bc5a898ddcaf9f09897-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 40f4f676921d6bc5a898ddcaf9f09897 columnFamilyName a 2024-11-24T08:29:40,930 DEBUG [StoreOpener-40f4f676921d6bc5a898ddcaf9f09897-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:29:40,930 INFO [StoreOpener-40f4f676921d6bc5a898ddcaf9f09897-1 {}] regionserver.HStore(327): Store=40f4f676921d6bc5a898ddcaf9f09897/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T08:29:40,931 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 40f4f676921d6bc5a898ddcaf9f09897 2024-11-24T08:29:40,932 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46165/hbase/data/default/testReplayEditsWrittenIntoWAL/40f4f676921d6bc5a898ddcaf9f09897 2024-11-24T08:29:40,935 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:46165/hbase/data/default/testReplayEditsWrittenIntoWAL/40f4f676921d6bc5a898ddcaf9f09897 2024-11-24T08:29:40,936 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:46165/hbase/data/default/testReplayEditsWrittenIntoWAL/40f4f676921d6bc5a898ddcaf9f09897/recovered.edits/0000000000000000002 2024-11-24T08:29:40,940 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:46165/hbase/data/default/testReplayEditsWrittenIntoWAL/40f4f676921d6bc5a898ddcaf9f09897/recovered.edits/0000000000000000002: isRecoveredEdits=true, hasTagCompression=true, hasValueCompression=false, valueCompressionType=null 2024-11-24T08:29:40,942 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 2, skipped 0, firstSequenceIdInLog=1, maxSequenceIdInLog=2, path=hdfs://localhost:46165/hbase/data/default/testReplayEditsWrittenIntoWAL/40f4f676921d6bc5a898ddcaf9f09897/recovered.edits/0000000000000000002 2024-11-24T08:29:40,942 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 40f4f676921d6bc5a898ddcaf9f09897 1/1 column families, dataSize=108 B heapSize=512 B 2024-11-24T08:29:40,967 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46165/hbase/data/default/testReplayEditsWrittenIntoWAL/40f4f676921d6bc5a898ddcaf9f09897/.tmp/a/5e18256df747486fbd300a4090c90252 is 58, key is testReplayEditsWrittenIntoWAL/a:1/1732436980765/Put/seqid=0 2024-11-24T08:29:40,975 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40533 is added to blk_1073741852_1028 (size=5170) 2024-11-24T08:29:40,975 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34459 is added to blk_1073741852_1028 (size=5170) 2024-11-24T08:29:40,975 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33555 is added to blk_1073741852_1028 (size=5170) 2024-11-24T08:29:40,977 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=108 B at sequenceid=2 (bloomFilter=true), to=hdfs://localhost:46165/hbase/data/default/testReplayEditsWrittenIntoWAL/40f4f676921d6bc5a898ddcaf9f09897/.tmp/a/5e18256df747486fbd300a4090c90252 2024-11-24T08:29:40,985 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46165/hbase/data/default/testReplayEditsWrittenIntoWAL/40f4f676921d6bc5a898ddcaf9f09897/.tmp/a/5e18256df747486fbd300a4090c90252 as hdfs://localhost:46165/hbase/data/default/testReplayEditsWrittenIntoWAL/40f4f676921d6bc5a898ddcaf9f09897/a/5e18256df747486fbd300a4090c90252 2024-11-24T08:29:40,992 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46165/hbase/data/default/testReplayEditsWrittenIntoWAL/40f4f676921d6bc5a898ddcaf9f09897/a/5e18256df747486fbd300a4090c90252, entries=2, sequenceid=2, filesize=5.0 K 2024-11-24T08:29:40,992 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~108 B/108, heapSize ~496 B/496, currentSize=0 B/0 for 40f4f676921d6bc5a898ddcaf9f09897 in 50ms, sequenceid=2, compaction requested=false; wal=null 2024-11-24T08:29:40,993 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:46165/hbase/data/default/testReplayEditsWrittenIntoWAL/40f4f676921d6bc5a898ddcaf9f09897/recovered.edits/0000000000000000002 2024-11-24T08:29:40,994 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 40f4f676921d6bc5a898ddcaf9f09897 2024-11-24T08:29:40,994 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 40f4f676921d6bc5a898ddcaf9f09897 2024-11-24T08:29:40,996 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 40f4f676921d6bc5a898ddcaf9f09897 2024-11-24T08:29:41,000 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:46165/hbase/data/default/testReplayEditsWrittenIntoWAL/40f4f676921d6bc5a898ddcaf9f09897/recovered.edits/2.seqid, newMaxSeqId=2, maxSeqId=1 2024-11-24T08:29:41,001 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 40f4f676921d6bc5a898ddcaf9f09897; next sequenceid=3; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=59752078, jitterRate=-0.10962465405464172}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-24T08:29:41,001 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 40f4f676921d6bc5a898ddcaf9f09897: Writing region info on filesystem at 1732436980925Initializing all the Stores at 1732436980927 (+2 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732436980927Obtaining lock to block concurrent updates at 1732436980942 (+15 ms)Preparing flush snapshotting stores in 40f4f676921d6bc5a898ddcaf9f09897 at 1732436980942Finished memstore snapshotting testReplayEditsWrittenIntoWAL,,1732436980693.40f4f676921d6bc5a898ddcaf9f09897., syncing WAL and waiting on mvcc, flushsize=dataSize=108, getHeapSize=496, getOffHeapSize=0, getCellsCount=2 at 1732436980942Flushing stores of testReplayEditsWrittenIntoWAL,,1732436980693.40f4f676921d6bc5a898ddcaf9f09897. at 1732436980942Flushing 40f4f676921d6bc5a898ddcaf9f09897/a: creating writer at 1732436980942Flushing 40f4f676921d6bc5a898ddcaf9f09897/a: appending metadata at 1732436980965 (+23 ms)Flushing 40f4f676921d6bc5a898ddcaf9f09897/a: closing flushed file at 1732436980965Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@1540b127: reopening flushed file at 1732436980984 (+19 ms)Finished flush of dataSize ~108 B/108, heapSize ~496 B/496, currentSize=0 B/0 for 40f4f676921d6bc5a898ddcaf9f09897 in 50ms, sequenceid=2, compaction requested=false; wal=null at 1732436980992 (+8 ms)Cleaning up temporary data from old regions at 1732436980994 (+2 ms)Region opened successfully at 1732436981001 (+7 ms) 2024-11-24T08:29:41,023 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestAsyncWALReplayCompressed#testNameConflictWhenSplit1 Thread=384 (was 374) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1541013681_22 at /127.0.0.1:34966 [Receiving block BP-1907186785-172.17.0.2-1732436973683:blk_1073741851_1027] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1541013681_22 at /127.0.0.1:43754 [Receiving block BP-1907186785-172.17.0.2-1732436973683:blk_1073741851_1027] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1541013681_22 at /127.0.0.1:38806 [Receiving block BP-1907186785-172.17.0.2-1732436973683:blk_1073741851_1027] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-10-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-10-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-10-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1541013681_22 at /127.0.0.1:35348 [Waiting for operation #13] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1541013681_22 at /127.0.0.1:54068 [Waiting for operation #10] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-1907186785-172.17.0.2-1732436973683:blk_1073741851_1027, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-1907186785-172.17.0.2-1732436973683:blk_1073741851_1027, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1541013681_22 at /127.0.0.1:43094 [Waiting for operation #14] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-1907186785-172.17.0.2-1732436973683:blk_1073741851_1027, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=785 (was 703) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=158 (was 158), ProcessCount=11 (was 11), AvailableMemoryMB=7117 (was 7124) 2024-11-24T08:29:41,037 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestAsyncWALReplayCompressed#testReplayEditsWrittenIntoWAL Thread=384, OpenFileDescriptor=785, MaxFileDescriptor=1048576, SystemLoadAverage=158, ProcessCount=11, AvailableMemoryMB=7116 2024-11-24T08:29:41,053 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-24T08:29:41,056 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-24T08:29:41,057 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-24T08:29:41,061 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-73192287, suffix=, logDir=hdfs://localhost:46165/hbase/WALs/hregion-73192287, archiveDir=hdfs://localhost:46165/hbase/oldWALs, maxLogs=32 2024-11-24T08:29:41,078 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-73192287/hregion-73192287.1732436981062, exclude list is [], retry=0 2024-11-24T08:29:41,082 DEBUG [AsyncFSWAL-12-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:34459,DS-6b4d1656-f49b-4a16-be49-2eb682c16e11,DISK] 2024-11-24T08:29:41,082 DEBUG [AsyncFSWAL-12-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40533,DS-5eeb9666-6dd5-4a48-9fb1-3f9aff1acdca,DISK] 2024-11-24T08:29:41,083 DEBUG [AsyncFSWAL-12-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33555,DS-8d97c3e2-1525-418c-a970-00a7157a0d31,DISK] 2024-11-24T08:29:41,088 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-73192287/hregion-73192287.1732436981062 2024-11-24T08:29:41,089 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:44131:44131),(127.0.0.1/127.0.0.1:42673:42673),(127.0.0.1/127.0.0.1:39357:39357)] 2024-11-24T08:29:41,089 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => ea08352c22404aa01e53b5118c9a8054, NAME => 'testReplayEditsWrittenIntoWAL,,1732436981054.ea08352c22404aa01e53b5118c9a8054.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testReplayEditsWrittenIntoWAL', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:46165/hbase 2024-11-24T08:29:41,100 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40533 is added to blk_1073741854_1030 (size=64) 2024-11-24T08:29:41,100 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34459 is added to blk_1073741854_1030 (size=64) 2024-11-24T08:29:41,101 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33555 is added to blk_1073741854_1030 (size=64) 2024-11-24T08:29:41,101 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenIntoWAL,,1732436981054.ea08352c22404aa01e53b5118c9a8054.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T08:29:41,103 INFO [StoreOpener-ea08352c22404aa01e53b5118c9a8054-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region ea08352c22404aa01e53b5118c9a8054 2024-11-24T08:29:41,105 INFO [StoreOpener-ea08352c22404aa01e53b5118c9a8054-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region ea08352c22404aa01e53b5118c9a8054 columnFamilyName a 2024-11-24T08:29:41,105 DEBUG [StoreOpener-ea08352c22404aa01e53b5118c9a8054-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:29:41,106 INFO [StoreOpener-ea08352c22404aa01e53b5118c9a8054-1 {}] regionserver.HStore(327): Store=ea08352c22404aa01e53b5118c9a8054/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T08:29:41,106 INFO [StoreOpener-ea08352c22404aa01e53b5118c9a8054-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region ea08352c22404aa01e53b5118c9a8054 2024-11-24T08:29:41,108 INFO [StoreOpener-ea08352c22404aa01e53b5118c9a8054-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region ea08352c22404aa01e53b5118c9a8054 columnFamilyName b 2024-11-24T08:29:41,108 DEBUG [StoreOpener-ea08352c22404aa01e53b5118c9a8054-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:29:41,108 INFO [StoreOpener-ea08352c22404aa01e53b5118c9a8054-1 {}] regionserver.HStore(327): Store=ea08352c22404aa01e53b5118c9a8054/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T08:29:41,108 INFO [StoreOpener-ea08352c22404aa01e53b5118c9a8054-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region ea08352c22404aa01e53b5118c9a8054 2024-11-24T08:29:41,110 INFO [StoreOpener-ea08352c22404aa01e53b5118c9a8054-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region ea08352c22404aa01e53b5118c9a8054 columnFamilyName c 2024-11-24T08:29:41,110 DEBUG [StoreOpener-ea08352c22404aa01e53b5118c9a8054-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:29:41,111 INFO [StoreOpener-ea08352c22404aa01e53b5118c9a8054-1 {}] regionserver.HStore(327): Store=ea08352c22404aa01e53b5118c9a8054/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T08:29:41,111 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for ea08352c22404aa01e53b5118c9a8054 2024-11-24T08:29:41,112 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46165/hbase/data/default/testReplayEditsWrittenIntoWAL/ea08352c22404aa01e53b5118c9a8054 2024-11-24T08:29:41,113 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46165/hbase/data/default/testReplayEditsWrittenIntoWAL/ea08352c22404aa01e53b5118c9a8054 2024-11-24T08:29:41,114 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for ea08352c22404aa01e53b5118c9a8054 2024-11-24T08:29:41,114 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for ea08352c22404aa01e53b5118c9a8054 2024-11-24T08:29:41,115 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsWrittenIntoWAL descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-24T08:29:41,116 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for ea08352c22404aa01e53b5118c9a8054 2024-11-24T08:29:41,119 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:46165/hbase/data/default/testReplayEditsWrittenIntoWAL/ea08352c22404aa01e53b5118c9a8054/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-24T08:29:41,120 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened ea08352c22404aa01e53b5118c9a8054; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=64713520, jitterRate=-0.03569340705871582}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-24T08:29:41,120 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for ea08352c22404aa01e53b5118c9a8054: Writing region info on filesystem at 1732436981102Initializing all the Stores at 1732436981103 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732436981103Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732436981103Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732436981103Cleaning up temporary data from old regions at 1732436981114 (+11 ms)Region opened successfully at 1732436981120 (+6 ms) 2024-11-24T08:29:41,120 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing ea08352c22404aa01e53b5118c9a8054, disabling compactions & flushes 2024-11-24T08:29:41,121 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsWrittenIntoWAL,,1732436981054.ea08352c22404aa01e53b5118c9a8054. 2024-11-24T08:29:41,121 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsWrittenIntoWAL,,1732436981054.ea08352c22404aa01e53b5118c9a8054. 2024-11-24T08:29:41,121 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsWrittenIntoWAL,,1732436981054.ea08352c22404aa01e53b5118c9a8054. after waiting 0 ms 2024-11-24T08:29:41,121 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsWrittenIntoWAL,,1732436981054.ea08352c22404aa01e53b5118c9a8054. 2024-11-24T08:29:41,121 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsWrittenIntoWAL,,1732436981054.ea08352c22404aa01e53b5118c9a8054. 2024-11-24T08:29:41,121 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for ea08352c22404aa01e53b5118c9a8054: Waiting for close lock at 1732436981120Disabling compacts and flushes for region at 1732436981120Disabling writes for close at 1732436981121 (+1 ms)Writing region close event to WAL at 1732436981121Closed at 1732436981121 2024-11-24T08:29:41,125 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40533 is added to blk_1073741853_1029 (size=93) 2024-11-24T08:29:41,125 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33555 is added to blk_1073741853_1029 (size=93) 2024-11-24T08:29:41,125 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34459 is added to blk_1073741853_1029 (size=93) 2024-11-24T08:29:41,129 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-11-24T08:29:41,129 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-73192287:(num 1732436981062) 2024-11-24T08:29:41,129 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-24T08:29:41,131 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:46165/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1732436981052, archiveDir=hdfs://localhost:46165/hbase/oldWALs, maxLogs=32 2024-11-24T08:29:41,146 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testreplayeditswrittenintowal-manual,16010,1732436981052/wal.1732436981132, exclude list is [], retry=0 2024-11-24T08:29:41,149 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:34459,DS-6b4d1656-f49b-4a16-be49-2eb682c16e11,DISK] 2024-11-24T08:29:41,150 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40533,DS-5eeb9666-6dd5-4a48-9fb1-3f9aff1acdca,DISK] 2024-11-24T08:29:41,151 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33555,DS-8d97c3e2-1525-418c-a970-00a7157a0d31,DISK] 2024-11-24T08:29:41,154 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testreplayeditswrittenintowal-manual,16010,1732436981052/wal.1732436981132 2024-11-24T08:29:41,156 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:44131:44131),(127.0.0.1/127.0.0.1:42673:42673),(127.0.0.1/127.0.0.1:39357:39357)] 2024-11-24T08:29:41,426 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:46165/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1732436981052/wal.1732436981132, size=0 (0bytes) 2024-11-24T08:29:41,426 WARN [Time-limited test {}] wal.WALSplitter(453): File hdfs://localhost:46165/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1732436981052/wal.1732436981132 might be still open, length is 0 2024-11-24T08:29:41,427 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:46165/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1732436981052/wal.1732436981132 2024-11-24T08:29:41,428 WARN [IPC Server handler 3 on default port 46165 {}] namenode.FSNamesystem(3854): DIR* NameSystem.internalReleaseLease: File /hbase/WALs/testreplayeditswrittenintowal-manual,16010,1732436981052/wal.1732436981132 has not been closed. Lease recovery is in progress. RecoveryId = 1032 for block blk_1073741855_1031 2024-11-24T08:29:41,429 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=0 on file=hdfs://localhost:46165/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1732436981052/wal.1732436981132 after 1ms 2024-11-24T08:29:41,747 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1541013681_22 at /127.0.0.1:34990 [Receiving block BP-1907186785-172.17.0.2-1732436973683:blk_1073741855_1031] {}] datanode.DataXceiver(331): 127.0.0.1:34459:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:34990 dst: /127.0.0.1:34459 java.io.InterruptedIOException: Interrupted while waiting for IO on channel java.nio.channels.SocketChannel[connected local=localhost/127.0.0.1:34459 remote=/127.0.0.1:34990]. Total timeout mills is 60000, 59631 millis timeout left. at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:350) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?] at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?] at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?] at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?] at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?] at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T08:29:41,748 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1541013681_22 at /127.0.0.1:43806 [Receiving block BP-1907186785-172.17.0.2-1732436973683:blk_1073741855_1031] {}] datanode.DataXceiver(331): 127.0.0.1:33555:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:43806 dst: /127.0.0.1:33555 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T08:29:41,748 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1541013681_22 at /127.0.0.1:38844 [Receiving block BP-1907186785-172.17.0.2-1732436973683:blk_1073741855_1031] {}] datanode.DataXceiver(331): 127.0.0.1:40533:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:38844 dst: /127.0.0.1:40533 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T08:29:41,757 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34459 is added to blk_1073741855_1032 (size=200589) 2024-11-24T08:29:41,759 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40533 is added to blk_1073741855_1032 (size=200589) 2024-11-24T08:29:41,760 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33555 is added to blk_1073741855_1032 (size=200589) 2024-11-24T08:29:45,209 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-24T08:29:45,255 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta' 2024-11-24T08:29:45,430 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=1 on file=hdfs://localhost:46165/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1732436981052/wal.1732436981132 after 4003ms 2024-11-24T08:29:45,434 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:46165/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1732436981052/wal.1732436981132: isRecoveredEdits=false, hasTagCompression=true, hasValueCompression=false, valueCompressionType=null 2024-11-24T08:29:45,436 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:46165/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1732436981052/wal.1732436981132 took 4010ms 2024-11-24T08:29:45,443 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000001-wal.1732436981132.temp 2024-11-24T08:29:45,444 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:46165/hbase/data/default/testReplayEditsWrittenIntoWAL/ea08352c22404aa01e53b5118c9a8054/recovered.edits/0000000000000000001-wal.1732436981132.temp 2024-11-24T08:29:45,546 INFO [Time-limited test {}] wal.WALSplitter(493): EOF from hdfs://localhost:46165/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1732436981052/wal.1732436981132; continuing. 2024-11-24T08:29:45,546 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:46165/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1732436981052/wal.1732436981132 so closing down 2024-11-24T08:29:45,546 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-24T08:29:45,549 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-11-24T08:29:45,552 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33555 is added to blk_1073741856_1033 (size=200597) 2024-11-24T08:29:45,552 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40533 is added to blk_1073741856_1033 (size=200597) 2024-11-24T08:29:45,552 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34459 is added to blk_1073741856_1033 (size=200597) 2024-11-24T08:29:45,553 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:46165/hbase/data/default/testReplayEditsWrittenIntoWAL/ea08352c22404aa01e53b5118c9a8054/recovered.edits/0000000000000000001-wal.1732436981132.temp (wrote 3002 edits, skipped 0 edits in 41 ms) 2024-11-24T08:29:45,555 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:46165/hbase/data/default/testReplayEditsWrittenIntoWAL/ea08352c22404aa01e53b5118c9a8054/recovered.edits/0000000000000000001-wal.1732436981132.temp to hdfs://localhost:46165/hbase/data/default/testReplayEditsWrittenIntoWAL/ea08352c22404aa01e53b5118c9a8054/recovered.edits/0000000000000003002 2024-11-24T08:29:45,555 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 3002 edits across 1 Regions in 117 ms; skipped=0; WAL=hdfs://localhost:46165/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1732436981052/wal.1732436981132, size=0, length=0, corrupted=false, cancelled=false 2024-11-24T08:29:45,555 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:46165/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1732436981052/wal.1732436981132, journal: Splitting hdfs://localhost:46165/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1732436981052/wal.1732436981132, size=0 (0bytes) at 1732436981426Creating recovered edits writer path=hdfs://localhost:46165/hbase/data/default/testReplayEditsWrittenIntoWAL/ea08352c22404aa01e53b5118c9a8054/recovered.edits/0000000000000000001-wal.1732436981132.temp at 1732436985444 (+4018 ms)Split 1024 edits, skipped 0 edits. at 1732436985503 (+59 ms)Split 2048 edits, skipped 0 edits. at 1732436985526 (+23 ms)Finishing writing output for hdfs://localhost:46165/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1732436981052/wal.1732436981132 so closing down at 1732436985546 (+20 ms)3 split writer threads finished at 1732436985549 (+3 ms)Closed recovered edits writer path=hdfs://localhost:46165/hbase/data/default/testReplayEditsWrittenIntoWAL/ea08352c22404aa01e53b5118c9a8054/recovered.edits/0000000000000000001-wal.1732436981132.temp (wrote 3002 edits, skipped 0 edits in 41 ms) at 1732436985553 (+4 ms)Rename recovered edits hdfs://localhost:46165/hbase/data/default/testReplayEditsWrittenIntoWAL/ea08352c22404aa01e53b5118c9a8054/recovered.edits/0000000000000000001-wal.1732436981132.temp to hdfs://localhost:46165/hbase/data/default/testReplayEditsWrittenIntoWAL/ea08352c22404aa01e53b5118c9a8054/recovered.edits/0000000000000003002 at 1732436985555 (+2 ms)Processed 3002 edits across 1 Regions in 117 ms; skipped=0; WAL=hdfs://localhost:46165/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1732436981052/wal.1732436981132, size=0, length=0, corrupted=false, cancelled=false at 1732436985555 2024-11-24T08:29:45,557 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:46165/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1732436981052/wal.1732436981132 to hdfs://localhost:46165/hbase/oldWALs/wal.1732436981132 2024-11-24T08:29:45,558 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:46165/hbase/data/default/testReplayEditsWrittenIntoWAL/ea08352c22404aa01e53b5118c9a8054/recovered.edits/0000000000000003002 2024-11-24T08:29:45,559 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-24T08:29:45,561 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:46165/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1732436981052, archiveDir=hdfs://localhost:46165/hbase/oldWALs, maxLogs=32 2024-11-24T08:29:45,575 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testreplayeditswrittenintowal-manual,16010,1732436981052/wal.1732436985561, exclude list is [], retry=0 2024-11-24T08:29:45,578 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:34459,DS-6b4d1656-f49b-4a16-be49-2eb682c16e11,DISK] 2024-11-24T08:29:45,579 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40533,DS-5eeb9666-6dd5-4a48-9fb1-3f9aff1acdca,DISK] 2024-11-24T08:29:45,579 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33555,DS-8d97c3e2-1525-418c-a970-00a7157a0d31,DISK] 2024-11-24T08:29:45,581 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testreplayeditswrittenintowal-manual,16010,1732436981052/wal.1732436985561 2024-11-24T08:29:45,582 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:44131:44131),(127.0.0.1/127.0.0.1:42673:42673),(127.0.0.1/127.0.0.1:39357:39357)] 2024-11-24T08:29:45,582 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenIntoWAL,,1732436981054.ea08352c22404aa01e53b5118c9a8054.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T08:29:45,584 INFO [StoreOpener-ea08352c22404aa01e53b5118c9a8054-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region ea08352c22404aa01e53b5118c9a8054 2024-11-24T08:29:45,585 INFO [StoreOpener-ea08352c22404aa01e53b5118c9a8054-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region ea08352c22404aa01e53b5118c9a8054 columnFamilyName a 2024-11-24T08:29:45,585 DEBUG [StoreOpener-ea08352c22404aa01e53b5118c9a8054-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:29:45,585 INFO [StoreOpener-ea08352c22404aa01e53b5118c9a8054-1 {}] regionserver.HStore(327): Store=ea08352c22404aa01e53b5118c9a8054/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T08:29:45,586 INFO [StoreOpener-ea08352c22404aa01e53b5118c9a8054-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region ea08352c22404aa01e53b5118c9a8054 2024-11-24T08:29:45,586 INFO [StoreOpener-ea08352c22404aa01e53b5118c9a8054-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region ea08352c22404aa01e53b5118c9a8054 columnFamilyName b 2024-11-24T08:29:45,586 DEBUG [StoreOpener-ea08352c22404aa01e53b5118c9a8054-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:29:45,587 INFO [StoreOpener-ea08352c22404aa01e53b5118c9a8054-1 {}] regionserver.HStore(327): Store=ea08352c22404aa01e53b5118c9a8054/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T08:29:45,587 INFO [StoreOpener-ea08352c22404aa01e53b5118c9a8054-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region ea08352c22404aa01e53b5118c9a8054 2024-11-24T08:29:45,588 INFO [StoreOpener-ea08352c22404aa01e53b5118c9a8054-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region ea08352c22404aa01e53b5118c9a8054 columnFamilyName c 2024-11-24T08:29:45,588 DEBUG [StoreOpener-ea08352c22404aa01e53b5118c9a8054-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:29:45,588 INFO [StoreOpener-ea08352c22404aa01e53b5118c9a8054-1 {}] regionserver.HStore(327): Store=ea08352c22404aa01e53b5118c9a8054/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T08:29:45,588 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for ea08352c22404aa01e53b5118c9a8054 2024-11-24T08:29:45,589 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46165/hbase/data/default/testReplayEditsWrittenIntoWAL/ea08352c22404aa01e53b5118c9a8054 2024-11-24T08:29:45,591 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:46165/hbase/data/default/testReplayEditsWrittenIntoWAL/ea08352c22404aa01e53b5118c9a8054 2024-11-24T08:29:45,592 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:46165/hbase/data/default/testReplayEditsWrittenIntoWAL/ea08352c22404aa01e53b5118c9a8054/recovered.edits/0000000000000003002 2024-11-24T08:29:45,595 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:46165/hbase/data/default/testReplayEditsWrittenIntoWAL/ea08352c22404aa01e53b5118c9a8054/recovered.edits/0000000000000003002: isRecoveredEdits=true, hasTagCompression=true, hasValueCompression=false, valueCompressionType=null 2024-11-24T08:29:45,625 INFO [Time-limited test {}] wal.AbstractTestWALReplay$4$1(796): InternalFlushCache Invoked 2024-11-24T08:29:46,004 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing ea08352c22404aa01e53b5118c9a8054 3/3 column families, dataSize=42.49 KB heapSize=100.11 KB 2024-11-24T08:29:46,042 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46165/hbase/data/default/testReplayEditsWrittenIntoWAL/ea08352c22404aa01e53b5118c9a8054/.tmp/a/3463b4beed65489c87fb56f15230158d is 62, key is testReplayEditsWrittenIntoWAL/a:100/1732436981166/Put/seqid=0 2024-11-24T08:29:46,055 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40533 is added to blk_1073741858_1035 (size=50463) 2024-11-24T08:29:46,056 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34459 is added to blk_1073741858_1035 (size=50463) 2024-11-24T08:29:46,056 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33555 is added to blk_1073741858_1035 (size=50463) 2024-11-24T08:29:46,057 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=754 (bloomFilter=true), to=hdfs://localhost:46165/hbase/data/default/testReplayEditsWrittenIntoWAL/ea08352c22404aa01e53b5118c9a8054/.tmp/a/3463b4beed65489c87fb56f15230158d 2024-11-24T08:29:46,065 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46165/hbase/data/default/testReplayEditsWrittenIntoWAL/ea08352c22404aa01e53b5118c9a8054/.tmp/a/3463b4beed65489c87fb56f15230158d as hdfs://localhost:46165/hbase/data/default/testReplayEditsWrittenIntoWAL/ea08352c22404aa01e53b5118c9a8054/a/3463b4beed65489c87fb56f15230158d 2024-11-24T08:29:46,071 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46165/hbase/data/default/testReplayEditsWrittenIntoWAL/ea08352c22404aa01e53b5118c9a8054/a/3463b4beed65489c87fb56f15230158d, entries=754, sequenceid=754, filesize=49.3 K 2024-11-24T08:29:46,072 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~42.49 KB/43512, heapSize ~99.59 KB/101984, currentSize=0 B/0 for ea08352c22404aa01e53b5118c9a8054 in 68ms, sequenceid=754, compaction requested=false; wal=null 2024-11-24T08:29:46,109 INFO [Time-limited test {}] wal.AbstractTestWALReplay$4$1(796): InternalFlushCache Invoked 2024-11-24T08:29:46,109 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing ea08352c22404aa01e53b5118c9a8054 3/3 column families, dataSize=42.49 KB heapSize=100.11 KB 2024-11-24T08:29:46,120 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46165/hbase/data/default/testReplayEditsWrittenIntoWAL/ea08352c22404aa01e53b5118c9a8054/.tmp/a/bc8664e6b1e3499fb439af51a7660405 is 62, key is testReplayEditsWrittenIntoWAL/a:754/1732436981219/Put/seqid=0 2024-11-24T08:29:46,131 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34459 is added to blk_1073741859_1036 (size=20072) 2024-11-24T08:29:46,132 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40533 is added to blk_1073741859_1036 (size=20072) 2024-11-24T08:29:46,133 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33555 is added to blk_1073741859_1036 (size=20072) 2024-11-24T08:29:46,137 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.93 KB at sequenceid=1508 (bloomFilter=true), to=hdfs://localhost:46165/hbase/data/default/testReplayEditsWrittenIntoWAL/ea08352c22404aa01e53b5118c9a8054/.tmp/a/bc8664e6b1e3499fb439af51a7660405 2024-11-24T08:29:46,171 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46165/hbase/data/default/testReplayEditsWrittenIntoWAL/ea08352c22404aa01e53b5118c9a8054/.tmp/b/012c4f84f597410b83e09c2d3744a016 is 62, key is testReplayEditsWrittenIntoWAL/b:100/1732436981248/Put/seqid=0 2024-11-24T08:29:46,180 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40533 is added to blk_1073741860_1037 (size=35835) 2024-11-24T08:29:46,180 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33555 is added to blk_1073741860_1037 (size=35835) 2024-11-24T08:29:46,181 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34459 is added to blk_1073741860_1037 (size=35835) 2024-11-24T08:29:46,182 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=28.56 KB at sequenceid=1508 (bloomFilter=true), to=hdfs://localhost:46165/hbase/data/default/testReplayEditsWrittenIntoWAL/ea08352c22404aa01e53b5118c9a8054/.tmp/b/012c4f84f597410b83e09c2d3744a016 2024-11-24T08:29:46,190 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46165/hbase/data/default/testReplayEditsWrittenIntoWAL/ea08352c22404aa01e53b5118c9a8054/.tmp/a/bc8664e6b1e3499fb439af51a7660405 as hdfs://localhost:46165/hbase/data/default/testReplayEditsWrittenIntoWAL/ea08352c22404aa01e53b5118c9a8054/a/bc8664e6b1e3499fb439af51a7660405 2024-11-24T08:29:46,197 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46165/hbase/data/default/testReplayEditsWrittenIntoWAL/ea08352c22404aa01e53b5118c9a8054/a/bc8664e6b1e3499fb439af51a7660405, entries=246, sequenceid=1508, filesize=19.6 K 2024-11-24T08:29:46,199 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46165/hbase/data/default/testReplayEditsWrittenIntoWAL/ea08352c22404aa01e53b5118c9a8054/.tmp/b/012c4f84f597410b83e09c2d3744a016 as hdfs://localhost:46165/hbase/data/default/testReplayEditsWrittenIntoWAL/ea08352c22404aa01e53b5118c9a8054/b/012c4f84f597410b83e09c2d3744a016 2024-11-24T08:29:46,207 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46165/hbase/data/default/testReplayEditsWrittenIntoWAL/ea08352c22404aa01e53b5118c9a8054/b/012c4f84f597410b83e09c2d3744a016, entries=508, sequenceid=1508, filesize=35.0 K 2024-11-24T08:29:46,208 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~42.49 KB/43512, heapSize ~99.83 KB/102224, currentSize=0 B/0 for ea08352c22404aa01e53b5118c9a8054 in 98ms, sequenceid=1508, compaction requested=false; wal=null 2024-11-24T08:29:46,223 INFO [Time-limited test {}] wal.AbstractTestWALReplay$4$1(796): InternalFlushCache Invoked 2024-11-24T08:29:46,224 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing ea08352c22404aa01e53b5118c9a8054 3/3 column families, dataSize=42.49 KB heapSize=100.11 KB 2024-11-24T08:29:46,234 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46165/hbase/data/default/testReplayEditsWrittenIntoWAL/ea08352c22404aa01e53b5118c9a8054/.tmp/b/89bf6e7c572c40c3b43e99358ba5d524 is 62, key is testReplayEditsWrittenIntoWAL/b:508/1732436981274/Put/seqid=0 2024-11-24T08:29:46,247 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34459 is added to blk_1073741861_1038 (size=35082) 2024-11-24T08:29:46,248 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33555 is added to blk_1073741861_1038 (size=35082) 2024-11-24T08:29:46,249 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40533 is added to blk_1073741861_1038 (size=35082) 2024-11-24T08:29:46,250 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=27.87 KB at sequenceid=2262 (bloomFilter=true), to=hdfs://localhost:46165/hbase/data/default/testReplayEditsWrittenIntoWAL/ea08352c22404aa01e53b5118c9a8054/.tmp/b/89bf6e7c572c40c3b43e99358ba5d524 2024-11-24T08:29:46,275 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46165/hbase/data/default/testReplayEditsWrittenIntoWAL/ea08352c22404aa01e53b5118c9a8054/.tmp/c/d87a21b4cf144ce1892d2ec1fff004f7 is 62, key is testReplayEditsWrittenIntoWAL/c:100/1732436981316/Put/seqid=0 2024-11-24T08:29:46,285 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40533 is added to blk_1073741862_1039 (size=20825) 2024-11-24T08:29:46,285 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33555 is added to blk_1073741862_1039 (size=20825) 2024-11-24T08:29:46,286 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34459 is added to blk_1073741862_1039 (size=20825) 2024-11-24T08:29:46,287 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=14.63 KB at sequenceid=2262 (bloomFilter=true), to=hdfs://localhost:46165/hbase/data/default/testReplayEditsWrittenIntoWAL/ea08352c22404aa01e53b5118c9a8054/.tmp/c/d87a21b4cf144ce1892d2ec1fff004f7 2024-11-24T08:29:46,295 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46165/hbase/data/default/testReplayEditsWrittenIntoWAL/ea08352c22404aa01e53b5118c9a8054/.tmp/b/89bf6e7c572c40c3b43e99358ba5d524 as hdfs://localhost:46165/hbase/data/default/testReplayEditsWrittenIntoWAL/ea08352c22404aa01e53b5118c9a8054/b/89bf6e7c572c40c3b43e99358ba5d524 2024-11-24T08:29:46,303 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46165/hbase/data/default/testReplayEditsWrittenIntoWAL/ea08352c22404aa01e53b5118c9a8054/b/89bf6e7c572c40c3b43e99358ba5d524, entries=492, sequenceid=2262, filesize=34.3 K 2024-11-24T08:29:46,304 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46165/hbase/data/default/testReplayEditsWrittenIntoWAL/ea08352c22404aa01e53b5118c9a8054/.tmp/c/d87a21b4cf144ce1892d2ec1fff004f7 as hdfs://localhost:46165/hbase/data/default/testReplayEditsWrittenIntoWAL/ea08352c22404aa01e53b5118c9a8054/c/d87a21b4cf144ce1892d2ec1fff004f7 2024-11-24T08:29:46,311 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46165/hbase/data/default/testReplayEditsWrittenIntoWAL/ea08352c22404aa01e53b5118c9a8054/c/d87a21b4cf144ce1892d2ec1fff004f7, entries=262, sequenceid=2262, filesize=20.3 K 2024-11-24T08:29:46,311 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~42.49 KB/43512, heapSize ~99.83 KB/102224, currentSize=0 B/0 for ea08352c22404aa01e53b5118c9a8054 in 88ms, sequenceid=2262, compaction requested=false; wal=null 2024-11-24T08:29:46,325 WARN [Time-limited test {}] regionserver.HRegion(5722): No family for cell testReplayEditsWrittenIntoWAL/another family:testReplayEditsWrittenIntoWAL/1732436981377/Put/vlen=29/seqid=0 in region testReplayEditsWrittenIntoWAL,,1732436981054.ea08352c22404aa01e53b5118c9a8054. 2024-11-24T08:29:46,330 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 3001, skipped 1, firstSequenceIdInLog=1, maxSequenceIdInLog=3002, path=hdfs://localhost:46165/hbase/data/default/testReplayEditsWrittenIntoWAL/ea08352c22404aa01e53b5118c9a8054/recovered.edits/0000000000000003002 2024-11-24T08:29:46,331 INFO [Time-limited test {}] wal.AbstractTestWALReplay$4$1(796): InternalFlushCache Invoked 2024-11-24T08:29:46,331 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing ea08352c22404aa01e53b5118c9a8054 3/3 column families, dataSize=41.85 KB heapSize=98.89 KB 2024-11-24T08:29:46,343 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46165/hbase/data/default/testReplayEditsWrittenIntoWAL/ea08352c22404aa01e53b5118c9a8054/.tmp/c/1839b6f760634f96ba591d183748887e is 62, key is testReplayEditsWrittenIntoWAL/c:262/1732436981326/Put/seqid=0 2024-11-24T08:29:46,351 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40533 is added to blk_1073741863_1040 (size=50301) 2024-11-24T08:29:46,351 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33555 is added to blk_1073741863_1040 (size=50301) 2024-11-24T08:29:46,352 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34459 is added to blk_1073741863_1040 (size=50301) 2024-11-24T08:29:46,353 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=41.85 KB at sequenceid=3002 (bloomFilter=true), to=hdfs://localhost:46165/hbase/data/default/testReplayEditsWrittenIntoWAL/ea08352c22404aa01e53b5118c9a8054/.tmp/c/1839b6f760634f96ba591d183748887e 2024-11-24T08:29:46,360 INFO [Time-limited test {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 1839b6f760634f96ba591d183748887e 2024-11-24T08:29:46,362 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46165/hbase/data/default/testReplayEditsWrittenIntoWAL/ea08352c22404aa01e53b5118c9a8054/.tmp/c/1839b6f760634f96ba591d183748887e as hdfs://localhost:46165/hbase/data/default/testReplayEditsWrittenIntoWAL/ea08352c22404aa01e53b5118c9a8054/c/1839b6f760634f96ba591d183748887e 2024-11-24T08:29:46,368 INFO [Time-limited test {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 1839b6f760634f96ba591d183748887e 2024-11-24T08:29:46,368 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46165/hbase/data/default/testReplayEditsWrittenIntoWAL/ea08352c22404aa01e53b5118c9a8054/c/1839b6f760634f96ba591d183748887e, entries=739, sequenceid=3002, filesize=49.1 K 2024-11-24T08:29:46,368 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~41.85 KB/42854, heapSize ~98.38 KB/100736, currentSize=0 B/0 for ea08352c22404aa01e53b5118c9a8054 in 37ms, sequenceid=3002, compaction requested=false; wal=null 2024-11-24T08:29:46,369 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:46165/hbase/data/default/testReplayEditsWrittenIntoWAL/ea08352c22404aa01e53b5118c9a8054/recovered.edits/0000000000000003002 2024-11-24T08:29:46,370 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for ea08352c22404aa01e53b5118c9a8054 2024-11-24T08:29:46,371 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for ea08352c22404aa01e53b5118c9a8054 2024-11-24T08:29:46,371 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsWrittenIntoWAL descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-24T08:29:46,373 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for ea08352c22404aa01e53b5118c9a8054 2024-11-24T08:29:46,375 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:46165/hbase/data/default/testReplayEditsWrittenIntoWAL/ea08352c22404aa01e53b5118c9a8054/recovered.edits/3002.seqid, newMaxSeqId=3002, maxSeqId=1 2024-11-24T08:29:46,376 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened ea08352c22404aa01e53b5118c9a8054; next sequenceid=3003; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=204800, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=74387334, jitterRate=0.10845765471458435}}}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-24T08:29:46,377 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for ea08352c22404aa01e53b5118c9a8054: Writing region info on filesystem at 1732436985582Initializing all the Stores at 1732436985583 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732436985583Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732436985584 (+1 ms)Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732436985584Cleaning up temporary data from old regions at 1732436986371 (+787 ms)Region opened successfully at 1732436986377 (+6 ms) 2024-11-24T08:29:46,439 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing ea08352c22404aa01e53b5118c9a8054, disabling compactions & flushes 2024-11-24T08:29:46,439 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsWrittenIntoWAL,,1732436981054.ea08352c22404aa01e53b5118c9a8054. 2024-11-24T08:29:46,439 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsWrittenIntoWAL,,1732436981054.ea08352c22404aa01e53b5118c9a8054. 2024-11-24T08:29:46,439 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsWrittenIntoWAL,,1732436981054.ea08352c22404aa01e53b5118c9a8054. after waiting 0 ms 2024-11-24T08:29:46,439 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsWrittenIntoWAL,,1732436981054.ea08352c22404aa01e53b5118c9a8054. 2024-11-24T08:29:46,442 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsWrittenIntoWAL,,1732436981054.ea08352c22404aa01e53b5118c9a8054. 2024-11-24T08:29:46,442 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for ea08352c22404aa01e53b5118c9a8054: Waiting for close lock at 1732436986439Disabling compacts and flushes for region at 1732436986439Disabling writes for close at 1732436986439Writing region close event to WAL at 1732436986442 (+3 ms)Closed at 1732436986442 2024-11-24T08:29:46,445 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40533 is added to blk_1073741857_1034 (size=93) 2024-11-24T08:29:46,446 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33555 is added to blk_1073741857_1034 (size=93) 2024-11-24T08:29:46,446 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34459 is added to blk_1073741857_1034 (size=93) 2024-11-24T08:29:46,449 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-11-24T08:29:46,449 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL wal:(num 1732436985561) 2024-11-24T08:29:46,463 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestAsyncWALReplayCompressed#testReplayEditsWrittenIntoWAL Thread=400 (was 384) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1995202469_22 at /127.0.0.1:35040 [Waiting for operation #6] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1995202469_22 at /127.0.0.1:38866 [Waiting for operation #17] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1995202469_22 at /127.0.0.1:43844 [Waiting for operation #15] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ed430c83-cb94-7d1a-6ba6-ccf0f0e534d9/cluster_264a055b-3bad-7287-6b90-a8fe44ac1a0a/data/data4 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ed430c83-cb94-7d1a-6ba6-ccf0f0e534d9/cluster_264a055b-3bad-7287-6b90-a8fe44ac1a0a/data/data3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Timer for 'HBase' metrics system java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.util.TimerThread.mainLoop(Timer.java:563) java.base@17.0.11/java.util.TimerThread.run(Timer.java:516) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ed430c83-cb94-7d1a-6ba6-ccf0f0e534d9/cluster_264a055b-3bad-7287-6b90-a8fe44ac1a0a/data/data1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ed430c83-cb94-7d1a-6ba6-ccf0f0e534d9/cluster_264a055b-3bad-7287-6b90-a8fe44ac1a0a/data/data2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Client (1987708253) connection to localhost/127.0.0.1:34731 from jenkins java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:40413 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704) java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903) app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-12-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-12-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: LeaseRenewer:jenkins.replay.wal.secondtime@localhost:46165 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Client (1987708253) connection to localhost/127.0.0.1:46165 from jenkins.replay.wal.secondtime java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:34731 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704) java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903) app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-12-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Client (1987708253) connection to localhost/127.0.0.1:40413 from jenkins java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1995202469_22 at /127.0.0.1:35050 [Waiting for operation #12] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=865 (was 785) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=153 (was 158), ProcessCount=11 (was 11), AvailableMemoryMB=7048 (was 7116) 2024-11-24T08:29:46,475 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestAsyncWALReplayCompressed#test2727 Thread=400, OpenFileDescriptor=865, MaxFileDescriptor=1048576, SystemLoadAverage=153, ProcessCount=11, AvailableMemoryMB=7047 2024-11-24T08:29:46,498 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-24T08:29:46,501 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-24T08:29:46,502 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-24T08:29:46,505 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-51550386, suffix=, logDir=hdfs://localhost:46165/hbase/WALs/hregion-51550386, archiveDir=hdfs://localhost:46165/hbase/oldWALs, maxLogs=32 2024-11-24T08:29:46,520 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-51550386/hregion-51550386.1732436986506, exclude list is [], retry=0 2024-11-24T08:29:46,523 DEBUG [AsyncFSWAL-14-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40533,DS-5eeb9666-6dd5-4a48-9fb1-3f9aff1acdca,DISK] 2024-11-24T08:29:46,523 DEBUG [AsyncFSWAL-14-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33555,DS-8d97c3e2-1525-418c-a970-00a7157a0d31,DISK] 2024-11-24T08:29:46,524 DEBUG [AsyncFSWAL-14-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:34459,DS-6b4d1656-f49b-4a16-be49-2eb682c16e11,DISK] 2024-11-24T08:29:46,526 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-51550386/hregion-51550386.1732436986506 2024-11-24T08:29:46,527 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:42673:42673),(127.0.0.1/127.0.0.1:39357:39357),(127.0.0.1/127.0.0.1:44131:44131)] 2024-11-24T08:29:46,527 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => 73b639ddf0ce9938260450104bfd0789, NAME => 'test2727,,1732436986499.73b639ddf0ce9938260450104bfd0789.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='test2727', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:46165/hbase 2024-11-24T08:29:46,538 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40533 is added to blk_1073741865_1042 (size=43) 2024-11-24T08:29:46,538 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34459 is added to blk_1073741865_1042 (size=43) 2024-11-24T08:29:46,539 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33555 is added to blk_1073741865_1042 (size=43) 2024-11-24T08:29:46,539 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated test2727,,1732436986499.73b639ddf0ce9938260450104bfd0789.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T08:29:46,541 INFO [StoreOpener-73b639ddf0ce9938260450104bfd0789-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 73b639ddf0ce9938260450104bfd0789 2024-11-24T08:29:46,543 INFO [StoreOpener-73b639ddf0ce9938260450104bfd0789-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 73b639ddf0ce9938260450104bfd0789 columnFamilyName a 2024-11-24T08:29:46,543 DEBUG [StoreOpener-73b639ddf0ce9938260450104bfd0789-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:29:46,543 INFO [StoreOpener-73b639ddf0ce9938260450104bfd0789-1 {}] regionserver.HStore(327): Store=73b639ddf0ce9938260450104bfd0789/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T08:29:46,544 INFO [StoreOpener-73b639ddf0ce9938260450104bfd0789-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 73b639ddf0ce9938260450104bfd0789 2024-11-24T08:29:46,545 INFO [StoreOpener-73b639ddf0ce9938260450104bfd0789-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 73b639ddf0ce9938260450104bfd0789 columnFamilyName b 2024-11-24T08:29:46,545 DEBUG [StoreOpener-73b639ddf0ce9938260450104bfd0789-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:29:46,546 INFO [StoreOpener-73b639ddf0ce9938260450104bfd0789-1 {}] regionserver.HStore(327): Store=73b639ddf0ce9938260450104bfd0789/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T08:29:46,546 INFO [StoreOpener-73b639ddf0ce9938260450104bfd0789-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 73b639ddf0ce9938260450104bfd0789 2024-11-24T08:29:46,548 INFO [StoreOpener-73b639ddf0ce9938260450104bfd0789-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 73b639ddf0ce9938260450104bfd0789 columnFamilyName c 2024-11-24T08:29:46,548 DEBUG [StoreOpener-73b639ddf0ce9938260450104bfd0789-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:29:46,548 INFO [StoreOpener-73b639ddf0ce9938260450104bfd0789-1 {}] regionserver.HStore(327): Store=73b639ddf0ce9938260450104bfd0789/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T08:29:46,548 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 73b639ddf0ce9938260450104bfd0789 2024-11-24T08:29:46,549 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46165/hbase/data/default/test2727/73b639ddf0ce9938260450104bfd0789 2024-11-24T08:29:46,549 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46165/hbase/data/default/test2727/73b639ddf0ce9938260450104bfd0789 2024-11-24T08:29:46,551 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 73b639ddf0ce9938260450104bfd0789 2024-11-24T08:29:46,551 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 73b639ddf0ce9938260450104bfd0789 2024-11-24T08:29:46,551 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table test2727 descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-24T08:29:46,553 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 73b639ddf0ce9938260450104bfd0789 2024-11-24T08:29:46,556 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:46165/hbase/data/default/test2727/73b639ddf0ce9938260450104bfd0789/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-24T08:29:46,556 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 73b639ddf0ce9938260450104bfd0789; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=66382704, jitterRate=-0.010820627212524414}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-24T08:29:46,558 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 73b639ddf0ce9938260450104bfd0789: Writing region info on filesystem at 1732436986539Initializing all the Stores at 1732436986540 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732436986541 (+1 ms)Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732436986541Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732436986541Cleaning up temporary data from old regions at 1732436986551 (+10 ms)Region opened successfully at 1732436986557 (+6 ms) 2024-11-24T08:29:46,558 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 73b639ddf0ce9938260450104bfd0789, disabling compactions & flushes 2024-11-24T08:29:46,558 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region test2727,,1732436986499.73b639ddf0ce9938260450104bfd0789. 2024-11-24T08:29:46,558 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on test2727,,1732436986499.73b639ddf0ce9938260450104bfd0789. 2024-11-24T08:29:46,558 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on test2727,,1732436986499.73b639ddf0ce9938260450104bfd0789. after waiting 0 ms 2024-11-24T08:29:46,558 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region test2727,,1732436986499.73b639ddf0ce9938260450104bfd0789. 2024-11-24T08:29:46,558 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed test2727,,1732436986499.73b639ddf0ce9938260450104bfd0789. 2024-11-24T08:29:46,559 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 73b639ddf0ce9938260450104bfd0789: Waiting for close lock at 1732436986558Disabling compacts and flushes for region at 1732436986558Disabling writes for close at 1732436986558Writing region close event to WAL at 1732436986558Closed at 1732436986558 2024-11-24T08:29:46,562 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33555 is added to blk_1073741864_1041 (size=93) 2024-11-24T08:29:46,563 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34459 is added to blk_1073741864_1041 (size=93) 2024-11-24T08:29:46,563 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40533 is added to blk_1073741864_1041 (size=93) 2024-11-24T08:29:46,564 WARN [Close-WAL-Writer-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(650): complete file /hbase/WALs/hregion-51550386/hregion-51550386.1732436986506 not finished, retry = 0 2024-11-24T08:29:46,667 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-11-24T08:29:46,667 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-51550386:(num 1732436986506) 2024-11-24T08:29:46,667 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-24T08:29:46,670 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:46165/hbase/WALs/test2727-manual,16010,1732436986497, archiveDir=hdfs://localhost:46165/hbase/oldWALs, maxLogs=32 2024-11-24T08:29:46,684 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/test2727-manual,16010,1732436986497/wal.1732436986670, exclude list is [], retry=0 2024-11-24T08:29:46,687 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33555,DS-8d97c3e2-1525-418c-a970-00a7157a0d31,DISK] 2024-11-24T08:29:46,688 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:34459,DS-6b4d1656-f49b-4a16-be49-2eb682c16e11,DISK] 2024-11-24T08:29:46,688 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40533,DS-5eeb9666-6dd5-4a48-9fb1-3f9aff1acdca,DISK] 2024-11-24T08:29:46,690 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/test2727-manual,16010,1732436986497/wal.1732436986670 2024-11-24T08:29:46,691 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:39357:39357),(127.0.0.1/127.0.0.1:44131:44131),(127.0.0.1/127.0.0.1:42673:42673)] 2024-11-24T08:29:46,834 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34459 is added to blk_1073741866_1043 (size=200357) 2024-11-24T08:29:46,834 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40533 is added to blk_1073741866_1043 (size=200357) 2024-11-24T08:29:46,835 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33555 is added to blk_1073741866_1043 (size=200357) 2024-11-24T08:29:46,857 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:46165/hbase/WALs/test2727-manual,16010,1732436986497/wal.1732436986670, size=195.7 K (200357bytes) 2024-11-24T08:29:46,857 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:46165/hbase/WALs/test2727-manual,16010,1732436986497/wal.1732436986670 2024-11-24T08:29:46,858 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:46165/hbase/WALs/test2727-manual,16010,1732436986497/wal.1732436986670 after 1ms 2024-11-24T08:29:46,861 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:46165/hbase/WALs/test2727-manual,16010,1732436986497/wal.1732436986670: isRecoveredEdits=false, hasTagCompression=true, hasValueCompression=false, valueCompressionType=null 2024-11-24T08:29:46,862 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:46165/hbase/WALs/test2727-manual,16010,1732436986497/wal.1732436986670 took 5ms 2024-11-24T08:29:46,867 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000001-wal.1732436986670.temp 2024-11-24T08:29:46,869 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:46165/hbase/data/default/test2727/73b639ddf0ce9938260450104bfd0789/recovered.edits/0000000000000000001-wal.1732436986670.temp 2024-11-24T08:29:46,926 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:46165/hbase/WALs/test2727-manual,16010,1732436986497/wal.1732436986670 so closing down 2024-11-24T08:29:46,926 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-24T08:29:46,927 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-11-24T08:29:46,930 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34459 is added to blk_1073741867_1044 (size=200357) 2024-11-24T08:29:46,930 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40533 is added to blk_1073741867_1044 (size=200357) 2024-11-24T08:29:46,930 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33555 is added to blk_1073741867_1044 (size=200357) 2024-11-24T08:29:46,931 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:46165/hbase/data/default/test2727/73b639ddf0ce9938260450104bfd0789/recovered.edits/0000000000000000001-wal.1732436986670.temp (wrote 3000 edits, skipped 0 edits in 22 ms) 2024-11-24T08:29:46,933 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:46165/hbase/data/default/test2727/73b639ddf0ce9938260450104bfd0789/recovered.edits/0000000000000000001-wal.1732436986670.temp to hdfs://localhost:46165/hbase/data/default/test2727/73b639ddf0ce9938260450104bfd0789/recovered.edits/0000000000000003000 2024-11-24T08:29:46,934 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 3000 edits across 1 Regions in 71 ms; skipped=0; WAL=hdfs://localhost:46165/hbase/WALs/test2727-manual,16010,1732436986497/wal.1732436986670, size=195.7 K, length=200357, corrupted=false, cancelled=false 2024-11-24T08:29:46,934 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:46165/hbase/WALs/test2727-manual,16010,1732436986497/wal.1732436986670, journal: Splitting hdfs://localhost:46165/hbase/WALs/test2727-manual,16010,1732436986497/wal.1732436986670, size=195.7 K (200357bytes) at 1732436986857Creating recovered edits writer path=hdfs://localhost:46165/hbase/data/default/test2727/73b639ddf0ce9938260450104bfd0789/recovered.edits/0000000000000000001-wal.1732436986670.temp at 1732436986869 (+12 ms)Split 1024 edits, skipped 0 edits. at 1732436986886 (+17 ms)Split 2048 edits, skipped 0 edits. at 1732436986906 (+20 ms)Finishing writing output for hdfs://localhost:46165/hbase/WALs/test2727-manual,16010,1732436986497/wal.1732436986670 so closing down at 1732436986926 (+20 ms)3 split writer threads finished at 1732436986927 (+1 ms)Closed recovered edits writer path=hdfs://localhost:46165/hbase/data/default/test2727/73b639ddf0ce9938260450104bfd0789/recovered.edits/0000000000000000001-wal.1732436986670.temp (wrote 3000 edits, skipped 0 edits in 22 ms) at 1732436986931 (+4 ms)Rename recovered edits hdfs://localhost:46165/hbase/data/default/test2727/73b639ddf0ce9938260450104bfd0789/recovered.edits/0000000000000000001-wal.1732436986670.temp to hdfs://localhost:46165/hbase/data/default/test2727/73b639ddf0ce9938260450104bfd0789/recovered.edits/0000000000000003000 at 1732436986933 (+2 ms)Processed 3000 edits across 1 Regions in 71 ms; skipped=0; WAL=hdfs://localhost:46165/hbase/WALs/test2727-manual,16010,1732436986497/wal.1732436986670, size=195.7 K, length=200357, corrupted=false, cancelled=false at 1732436986934 (+1 ms) 2024-11-24T08:29:46,936 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:46165/hbase/WALs/test2727-manual,16010,1732436986497/wal.1732436986670 to hdfs://localhost:46165/hbase/oldWALs/wal.1732436986670 2024-11-24T08:29:46,937 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:46165/hbase/data/default/test2727/73b639ddf0ce9938260450104bfd0789/recovered.edits/0000000000000003000 2024-11-24T08:29:46,937 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-24T08:29:46,946 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:46165/hbase/WALs/test2727-manual,16010,1732436986497, archiveDir=hdfs://localhost:46165/hbase/oldWALs, maxLogs=32 2024-11-24T08:29:46,962 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/test2727-manual,16010,1732436986497/wal.1732436986947, exclude list is [], retry=0 2024-11-24T08:29:46,965 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40533,DS-5eeb9666-6dd5-4a48-9fb1-3f9aff1acdca,DISK] 2024-11-24T08:29:46,966 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:34459,DS-6b4d1656-f49b-4a16-be49-2eb682c16e11,DISK] 2024-11-24T08:29:46,966 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33555,DS-8d97c3e2-1525-418c-a970-00a7157a0d31,DISK] 2024-11-24T08:29:46,968 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/test2727-manual,16010,1732436986497/wal.1732436986947 2024-11-24T08:29:46,968 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:42673:42673),(127.0.0.1/127.0.0.1:44131:44131),(127.0.0.1/127.0.0.1:39357:39357)] 2024-11-24T08:29:47,131 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34459 is added to blk_1073741868_1045 (size=200484) 2024-11-24T08:29:47,131 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40533 is added to blk_1073741868_1045 (size=200484) 2024-11-24T08:29:47,132 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33555 is added to blk_1073741868_1045 (size=200484) 2024-11-24T08:29:47,151 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:46165/hbase/WALs/test2727-manual,16010,1732436986497/wal.1732436986947, size=195.8 K (200484bytes) 2024-11-24T08:29:47,151 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:46165/hbase/WALs/test2727-manual,16010,1732436986497/wal.1732436986947 2024-11-24T08:29:47,151 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:46165/hbase/WALs/test2727-manual,16010,1732436986497/wal.1732436986947 after 0ms 2024-11-24T08:29:47,155 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:46165/hbase/WALs/test2727-manual,16010,1732436986497/wal.1732436986947: isRecoveredEdits=false, hasTagCompression=true, hasValueCompression=false, valueCompressionType=null 2024-11-24T08:29:47,157 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:46165/hbase/WALs/test2727-manual,16010,1732436986497/wal.1732436986947 took 6ms 2024-11-24T08:29:47,162 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000003001-wal.1732436986947.temp 2024-11-24T08:29:47,164 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:46165/hbase/data/default/test2727/73b639ddf0ce9938260450104bfd0789/recovered.edits/0000000000000003001-wal.1732436986947.temp 2024-11-24T08:29:47,221 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:46165/hbase/WALs/test2727-manual,16010,1732436986497/wal.1732436986947 so closing down 2024-11-24T08:29:47,221 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-24T08:29:47,221 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-11-24T08:29:47,224 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34459 is added to blk_1073741869_1046 (size=200484) 2024-11-24T08:29:47,224 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40533 is added to blk_1073741869_1046 (size=200484) 2024-11-24T08:29:47,225 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33555 is added to blk_1073741869_1046 (size=200484) 2024-11-24T08:29:47,225 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:46165/hbase/data/default/test2727/73b639ddf0ce9938260450104bfd0789/recovered.edits/0000000000000003001-wal.1732436986947.temp (wrote 3000 edits, skipped 0 edits in 20 ms) 2024-11-24T08:29:47,227 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:46165/hbase/data/default/test2727/73b639ddf0ce9938260450104bfd0789/recovered.edits/0000000000000003001-wal.1732436986947.temp to hdfs://localhost:46165/hbase/data/default/test2727/73b639ddf0ce9938260450104bfd0789/recovered.edits/0000000000000006000 2024-11-24T08:29:47,228 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 3000 edits across 1 Regions in 69 ms; skipped=0; WAL=hdfs://localhost:46165/hbase/WALs/test2727-manual,16010,1732436986497/wal.1732436986947, size=195.8 K, length=200484, corrupted=false, cancelled=false 2024-11-24T08:29:47,228 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:46165/hbase/WALs/test2727-manual,16010,1732436986497/wal.1732436986947, journal: Splitting hdfs://localhost:46165/hbase/WALs/test2727-manual,16010,1732436986497/wal.1732436986947, size=195.8 K (200484bytes) at 1732436987151Creating recovered edits writer path=hdfs://localhost:46165/hbase/data/default/test2727/73b639ddf0ce9938260450104bfd0789/recovered.edits/0000000000000003001-wal.1732436986947.temp at 1732436987164 (+13 ms)Split 1024 edits, skipped 0 edits. at 1732436987181 (+17 ms)Split 2048 edits, skipped 0 edits. at 1732436987202 (+21 ms)Finishing writing output for hdfs://localhost:46165/hbase/WALs/test2727-manual,16010,1732436986497/wal.1732436986947 so closing down at 1732436987221 (+19 ms)3 split writer threads finished at 1732436987221Closed recovered edits writer path=hdfs://localhost:46165/hbase/data/default/test2727/73b639ddf0ce9938260450104bfd0789/recovered.edits/0000000000000003001-wal.1732436986947.temp (wrote 3000 edits, skipped 0 edits in 20 ms) at 1732436987226 (+5 ms)Rename recovered edits hdfs://localhost:46165/hbase/data/default/test2727/73b639ddf0ce9938260450104bfd0789/recovered.edits/0000000000000003001-wal.1732436986947.temp to hdfs://localhost:46165/hbase/data/default/test2727/73b639ddf0ce9938260450104bfd0789/recovered.edits/0000000000000006000 at 1732436987227 (+1 ms)Processed 3000 edits across 1 Regions in 69 ms; skipped=0; WAL=hdfs://localhost:46165/hbase/WALs/test2727-manual,16010,1732436986497/wal.1732436986947, size=195.8 K, length=200484, corrupted=false, cancelled=false at 1732436987228 (+1 ms) 2024-11-24T08:29:47,230 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:46165/hbase/WALs/test2727-manual,16010,1732436986497/wal.1732436986947 to hdfs://localhost:46165/hbase/oldWALs/wal.1732436986947 2024-11-24T08:29:47,231 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:46165/hbase/data/default/test2727/73b639ddf0ce9938260450104bfd0789/recovered.edits/0000000000000006000 2024-11-24T08:29:47,231 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-24T08:29:47,233 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:46165/hbase/WALs/test2727-manual,16010,1732436986497, archiveDir=hdfs://localhost:46165/hbase/oldWALs, maxLogs=32 2024-11-24T08:29:47,249 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/test2727-manual,16010,1732436986497/wal.1732436987234, exclude list is [], retry=0 2024-11-24T08:29:47,253 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33555,DS-8d97c3e2-1525-418c-a970-00a7157a0d31,DISK] 2024-11-24T08:29:47,253 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40533,DS-5eeb9666-6dd5-4a48-9fb1-3f9aff1acdca,DISK] 2024-11-24T08:29:47,254 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:34459,DS-6b4d1656-f49b-4a16-be49-2eb682c16e11,DISK] 2024-11-24T08:29:47,256 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/test2727-manual,16010,1732436986497/wal.1732436987234 2024-11-24T08:29:47,256 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:39357:39357),(127.0.0.1/127.0.0.1:42673:42673),(127.0.0.1/127.0.0.1:44131:44131)] 2024-11-24T08:29:47,256 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 73b639ddf0ce9938260450104bfd0789, NAME => 'test2727,,1732436986499.73b639ddf0ce9938260450104bfd0789.', STARTKEY => '', ENDKEY => ''} 2024-11-24T08:29:47,257 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated test2727,,1732436986499.73b639ddf0ce9938260450104bfd0789.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T08:29:47,257 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 73b639ddf0ce9938260450104bfd0789 2024-11-24T08:29:47,257 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 73b639ddf0ce9938260450104bfd0789 2024-11-24T08:29:47,258 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta 2024-11-24T08:29:47,259 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_meta Metrics about Tables on a single HBase RegionServer 2024-11-24T08:29:47,259 INFO [StoreOpener-73b639ddf0ce9938260450104bfd0789-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 73b639ddf0ce9938260450104bfd0789 2024-11-24T08:29:47,260 INFO [StoreOpener-73b639ddf0ce9938260450104bfd0789-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 73b639ddf0ce9938260450104bfd0789 columnFamilyName a 2024-11-24T08:29:47,260 DEBUG [StoreOpener-73b639ddf0ce9938260450104bfd0789-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:29:47,261 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-24T08:29:47,261 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint Metrics about HBase RegionObservers 2024-11-24T08:29:47,261 INFO [StoreOpener-73b639ddf0ce9938260450104bfd0789-1 {}] regionserver.HStore(327): Store=73b639ddf0ce9938260450104bfd0789/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T08:29:47,261 INFO [StoreOpener-73b639ddf0ce9938260450104bfd0789-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 73b639ddf0ce9938260450104bfd0789 2024-11-24T08:29:47,261 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_master_table_store 2024-11-24T08:29:47,261 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_master_table_store Metrics about Tables on a single HBase RegionServer 2024-11-24T08:29:47,262 INFO [StoreOpener-73b639ddf0ce9938260450104bfd0789-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 73b639ddf0ce9938260450104bfd0789 columnFamilyName b 2024-11-24T08:29:47,262 DEBUG [StoreOpener-73b639ddf0ce9938260450104bfd0789-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:29:47,263 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testReplayEditsWrittenIntoWAL 2024-11-24T08:29:47,263 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testReplayEditsWrittenIntoWAL Metrics about Tables on a single HBase RegionServer 2024-11-24T08:29:47,263 INFO [StoreOpener-73b639ddf0ce9938260450104bfd0789-1 {}] regionserver.HStore(327): Store=73b639ddf0ce9938260450104bfd0789/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T08:29:47,263 INFO [StoreOpener-73b639ddf0ce9938260450104bfd0789-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 73b639ddf0ce9938260450104bfd0789 2024-11-24T08:29:47,264 INFO [StoreOpener-73b639ddf0ce9938260450104bfd0789-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 73b639ddf0ce9938260450104bfd0789 columnFamilyName c 2024-11-24T08:29:47,264 DEBUG [StoreOpener-73b639ddf0ce9938260450104bfd0789-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:29:47,265 INFO [StoreOpener-73b639ddf0ce9938260450104bfd0789-1 {}] regionserver.HStore(327): Store=73b639ddf0ce9938260450104bfd0789/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T08:29:47,265 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 73b639ddf0ce9938260450104bfd0789 2024-11-24T08:29:47,266 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46165/hbase/data/default/test2727/73b639ddf0ce9938260450104bfd0789 2024-11-24T08:29:47,268 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 2 recovered edits file(s) under hdfs://localhost:46165/hbase/data/default/test2727/73b639ddf0ce9938260450104bfd0789 2024-11-24T08:29:47,269 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:46165/hbase/data/default/test2727/73b639ddf0ce9938260450104bfd0789/recovered.edits/0000000000000003000 2024-11-24T08:29:47,272 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:46165/hbase/data/default/test2727/73b639ddf0ce9938260450104bfd0789/recovered.edits/0000000000000003000: isRecoveredEdits=true, hasTagCompression=true, hasValueCompression=false, valueCompressionType=null 2024-11-24T08:29:47,340 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 3000, skipped 0, firstSequenceIdInLog=1, maxSequenceIdInLog=3000, path=hdfs://localhost:46165/hbase/data/default/test2727/73b639ddf0ce9938260450104bfd0789/recovered.edits/0000000000000003000 2024-11-24T08:29:47,341 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:46165/hbase/data/default/test2727/73b639ddf0ce9938260450104bfd0789/recovered.edits/0000000000000006000 2024-11-24T08:29:47,345 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:46165/hbase/data/default/test2727/73b639ddf0ce9938260450104bfd0789/recovered.edits/0000000000000006000: isRecoveredEdits=true, hasTagCompression=true, hasValueCompression=false, valueCompressionType=null 2024-11-24T08:29:47,406 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 3000, skipped 0, firstSequenceIdInLog=3001, maxSequenceIdInLog=6000, path=hdfs://localhost:46165/hbase/data/default/test2727/73b639ddf0ce9938260450104bfd0789/recovered.edits/0000000000000006000 2024-11-24T08:29:47,406 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 73b639ddf0ce9938260450104bfd0789 3/3 column families, dataSize=215.51 KB heapSize=657 KB 2024-11-24T08:29:47,431 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46165/hbase/data/default/test2727/73b639ddf0ce9938260450104bfd0789/.tmp/a/6acbcfe3327d49fab73485c151dc5bb0 is 41, key is test2727/a:100/1732436986973/Put/seqid=0 2024-11-24T08:29:47,440 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33555 is added to blk_1073741871_1048 (size=84227) 2024-11-24T08:29:47,441 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34459 is added to blk_1073741871_1048 (size=84227) 2024-11-24T08:29:47,441 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40533 is added to blk_1073741871_1048 (size=84227) 2024-11-24T08:29:47,441 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=71.84 KB at sequenceid=6000 (bloomFilter=true), to=hdfs://localhost:46165/hbase/data/default/test2727/73b639ddf0ce9938260450104bfd0789/.tmp/a/6acbcfe3327d49fab73485c151dc5bb0 2024-11-24T08:29:47,473 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46165/hbase/data/default/test2727/73b639ddf0ce9938260450104bfd0789/.tmp/b/c4346f493be94408bbbc267594922e6e is 41, key is test2727/b:100/1732436987022/Put/seqid=0 2024-11-24T08:29:47,478 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40533 is added to blk_1073741872_1049 (size=84609) 2024-11-24T08:29:47,479 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34459 is added to blk_1073741872_1049 (size=84609) 2024-11-24T08:29:47,480 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33555 is added to blk_1073741872_1049 (size=84609) 2024-11-24T08:29:47,480 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=71.84 KB at sequenceid=6000 (bloomFilter=true), to=hdfs://localhost:46165/hbase/data/default/test2727/73b639ddf0ce9938260450104bfd0789/.tmp/b/c4346f493be94408bbbc267594922e6e 2024-11-24T08:29:47,507 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46165/hbase/data/default/test2727/73b639ddf0ce9938260450104bfd0789/.tmp/c/c7b4ddd9876d4412b96b9dacd1e7ab75 is 41, key is test2727/c:100/1732436987086/Put/seqid=0 2024-11-24T08:29:47,515 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40533 is added to blk_1073741873_1050 (size=84609) 2024-11-24T08:29:47,515 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34459 is added to blk_1073741873_1050 (size=84609) 2024-11-24T08:29:47,516 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33555 is added to blk_1073741873_1050 (size=84609) 2024-11-24T08:29:47,516 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=71.84 KB at sequenceid=6000 (bloomFilter=true), to=hdfs://localhost:46165/hbase/data/default/test2727/73b639ddf0ce9938260450104bfd0789/.tmp/c/c7b4ddd9876d4412b96b9dacd1e7ab75 2024-11-24T08:29:47,524 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46165/hbase/data/default/test2727/73b639ddf0ce9938260450104bfd0789/.tmp/a/6acbcfe3327d49fab73485c151dc5bb0 as hdfs://localhost:46165/hbase/data/default/test2727/73b639ddf0ce9938260450104bfd0789/a/6acbcfe3327d49fab73485c151dc5bb0 2024-11-24T08:29:47,531 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46165/hbase/data/default/test2727/73b639ddf0ce9938260450104bfd0789/a/6acbcfe3327d49fab73485c151dc5bb0, entries=2000, sequenceid=6000, filesize=82.3 K 2024-11-24T08:29:47,532 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46165/hbase/data/default/test2727/73b639ddf0ce9938260450104bfd0789/.tmp/b/c4346f493be94408bbbc267594922e6e as hdfs://localhost:46165/hbase/data/default/test2727/73b639ddf0ce9938260450104bfd0789/b/c4346f493be94408bbbc267594922e6e 2024-11-24T08:29:47,539 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46165/hbase/data/default/test2727/73b639ddf0ce9938260450104bfd0789/b/c4346f493be94408bbbc267594922e6e, entries=2000, sequenceid=6000, filesize=82.6 K 2024-11-24T08:29:47,540 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46165/hbase/data/default/test2727/73b639ddf0ce9938260450104bfd0789/.tmp/c/c7b4ddd9876d4412b96b9dacd1e7ab75 as hdfs://localhost:46165/hbase/data/default/test2727/73b639ddf0ce9938260450104bfd0789/c/c7b4ddd9876d4412b96b9dacd1e7ab75 2024-11-24T08:29:47,547 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46165/hbase/data/default/test2727/73b639ddf0ce9938260450104bfd0789/c/c7b4ddd9876d4412b96b9dacd1e7ab75, entries=2000, sequenceid=6000, filesize=82.6 K 2024-11-24T08:29:47,547 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~215.51 KB/220680, heapSize ~656.95 KB/672720, currentSize=0 B/0 for 73b639ddf0ce9938260450104bfd0789 in 141ms, sequenceid=6000, compaction requested=false; wal=null 2024-11-24T08:29:47,548 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:46165/hbase/data/default/test2727/73b639ddf0ce9938260450104bfd0789/recovered.edits/0000000000000003000 2024-11-24T08:29:47,549 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:46165/hbase/data/default/test2727/73b639ddf0ce9938260450104bfd0789/recovered.edits/0000000000000006000 2024-11-24T08:29:47,550 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 73b639ddf0ce9938260450104bfd0789 2024-11-24T08:29:47,550 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 73b639ddf0ce9938260450104bfd0789 2024-11-24T08:29:47,551 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table test2727 descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-24T08:29:47,553 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 73b639ddf0ce9938260450104bfd0789 2024-11-24T08:29:47,556 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:46165/hbase/data/default/test2727/73b639ddf0ce9938260450104bfd0789/recovered.edits/6000.seqid, newMaxSeqId=6000, maxSeqId=1 2024-11-24T08:29:47,557 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 73b639ddf0ce9938260450104bfd0789; next sequenceid=6001; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=67254851, jitterRate=0.002175375819206238}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-24T08:29:47,559 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 73b639ddf0ce9938260450104bfd0789: Writing region info on filesystem at 1732436987257Initializing all the Stores at 1732436987258 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732436987258Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732436987259 (+1 ms)Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732436987259Obtaining lock to block concurrent updates at 1732436987406 (+147 ms)Preparing flush snapshotting stores in 73b639ddf0ce9938260450104bfd0789 at 1732436987406Finished memstore snapshotting test2727,,1732436986499.73b639ddf0ce9938260450104bfd0789., syncing WAL and waiting on mvcc, flushsize=dataSize=220680, getHeapSize=672720, getOffHeapSize=0, getCellsCount=6000 at 1732436987406Flushing stores of test2727,,1732436986499.73b639ddf0ce9938260450104bfd0789. at 1732436987406Flushing 73b639ddf0ce9938260450104bfd0789/a: creating writer at 1732436987407 (+1 ms)Flushing 73b639ddf0ce9938260450104bfd0789/a: appending metadata at 1732436987430 (+23 ms)Flushing 73b639ddf0ce9938260450104bfd0789/a: closing flushed file at 1732436987430Flushing 73b639ddf0ce9938260450104bfd0789/b: creating writer at 1732436987448 (+18 ms)Flushing 73b639ddf0ce9938260450104bfd0789/b: appending metadata at 1732436987471 (+23 ms)Flushing 73b639ddf0ce9938260450104bfd0789/b: closing flushed file at 1732436987471Flushing 73b639ddf0ce9938260450104bfd0789/c: creating writer at 1732436987487 (+16 ms)Flushing 73b639ddf0ce9938260450104bfd0789/c: appending metadata at 1732436987506 (+19 ms)Flushing 73b639ddf0ce9938260450104bfd0789/c: closing flushed file at 1732436987506Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@472c32fa: reopening flushed file at 1732436987523 (+17 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@33df1fc8: reopening flushed file at 1732436987531 (+8 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@28a6eedc: reopening flushed file at 1732436987539 (+8 ms)Finished flush of dataSize ~215.51 KB/220680, heapSize ~656.95 KB/672720, currentSize=0 B/0 for 73b639ddf0ce9938260450104bfd0789 in 141ms, sequenceid=6000, compaction requested=false; wal=null at 1732436987547 (+8 ms)Cleaning up temporary data from old regions at 1732436987550 (+3 ms)Region opened successfully at 1732436987559 (+9 ms) 2024-11-24T08:29:47,560 DEBUG [Time-limited test {}] wal.AbstractTestWALReplay(320): region.getOpenSeqNum(): 6001, wal3.id: 0 2024-11-24T08:29:47,561 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 73b639ddf0ce9938260450104bfd0789, disabling compactions & flushes 2024-11-24T08:29:47,561 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region test2727,,1732436986499.73b639ddf0ce9938260450104bfd0789. 2024-11-24T08:29:47,561 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on test2727,,1732436986499.73b639ddf0ce9938260450104bfd0789. 2024-11-24T08:29:47,561 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on test2727,,1732436986499.73b639ddf0ce9938260450104bfd0789. after waiting 0 ms 2024-11-24T08:29:47,561 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region test2727,,1732436986499.73b639ddf0ce9938260450104bfd0789. 2024-11-24T08:29:47,562 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed test2727,,1732436986499.73b639ddf0ce9938260450104bfd0789. 2024-11-24T08:29:47,563 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 73b639ddf0ce9938260450104bfd0789: Waiting for close lock at 1732436987560Disabling compacts and flushes for region at 1732436987560Disabling writes for close at 1732436987561 (+1 ms)Writing region close event to WAL at 1732436987562 (+1 ms)Closed at 1732436987562 2024-11-24T08:29:47,566 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34459 is added to blk_1073741870_1047 (size=93) 2024-11-24T08:29:47,566 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33555 is added to blk_1073741870_1047 (size=93) 2024-11-24T08:29:47,568 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40533 is added to blk_1073741870_1047 (size=93) 2024-11-24T08:29:47,575 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-11-24T08:29:47,575 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL wal:(num 1732436987234) 2024-11-24T08:29:47,588 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestAsyncWALReplayCompressed#test2727 Thread=402 (was 400) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1541013681_22 at /127.0.0.1:35226 [Waiting for operation #9] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-14-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1541013681_22 at /127.0.0.1:39002 [Waiting for operation #9] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-14-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-14-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=927 (was 865) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=153 (was 153), ProcessCount=11 (was 11), AvailableMemoryMB=6831 (was 7047) 2024-11-24T08:29:47,599 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestAsyncWALReplayCompressed#testSequentialEditLogSeqNum Thread=402, OpenFileDescriptor=927, MaxFileDescriptor=1048576, SystemLoadAverage=153, ProcessCount=11, AvailableMemoryMB=6830 2024-11-24T08:29:47,616 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-24T08:29:47,623 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:46165/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1732436987615, archiveDir=hdfs://localhost:46165/hbase/oldWALs, maxLogs=32 2024-11-24T08:29:47,624 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732436987624 2024-11-24T08:29:47,633 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testsequentialeditlogseqnum-manual,16010,1732436987615/wal.1732436987624 2024-11-24T08:29:47,635 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new MockWAL writer with pipeline: [(127.0.0.1/127.0.0.1:42673:42673),(127.0.0.1/127.0.0.1:44131:44131),(127.0.0.1/127.0.0.1:39357:39357)] 2024-11-24T08:29:47,636 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => a9b3cedf953393869a4d86b0d4e9453c, NAME => 'testSequentialEditLogSeqNum,,1732436987617.a9b3cedf953393869a4d86b0d4e9453c.', STARTKEY => '', ENDKEY => ''} 2024-11-24T08:29:47,637 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testSequentialEditLogSeqNum,,1732436987617.a9b3cedf953393869a4d86b0d4e9453c.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T08:29:47,637 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for a9b3cedf953393869a4d86b0d4e9453c 2024-11-24T08:29:47,637 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for a9b3cedf953393869a4d86b0d4e9453c 2024-11-24T08:29:47,638 WARN [Time-limited test {}] regionserver.HRegionFileSystem(836): hdfs://localhost:46165/hbase/data/default/testSequentialEditLogSeqNum/a9b3cedf953393869a4d86b0d4e9453c doesn't exist for region: a9b3cedf953393869a4d86b0d4e9453c on table testSequentialEditLogSeqNum 2024-11-24T08:29:47,639 WARN [Time-limited test {}] regionserver.HRegionFileSystem(854): .regioninfo file not found for region: a9b3cedf953393869a4d86b0d4e9453c on table testSequentialEditLogSeqNum 2024-11-24T08:29:47,647 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33555 is added to blk_1073741875_1052 (size=62) 2024-11-24T08:29:47,648 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40533 is added to blk_1073741875_1052 (size=62) 2024-11-24T08:29:47,648 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34459 is added to blk_1073741875_1052 (size=62) 2024-11-24T08:29:47,650 INFO [StoreOpener-a9b3cedf953393869a4d86b0d4e9453c-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region a9b3cedf953393869a4d86b0d4e9453c 2024-11-24T08:29:47,652 INFO [StoreOpener-a9b3cedf953393869a4d86b0d4e9453c-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region a9b3cedf953393869a4d86b0d4e9453c columnFamilyName a 2024-11-24T08:29:47,652 DEBUG [StoreOpener-a9b3cedf953393869a4d86b0d4e9453c-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:29:47,653 INFO [StoreOpener-a9b3cedf953393869a4d86b0d4e9453c-1 {}] regionserver.HStore(327): Store=a9b3cedf953393869a4d86b0d4e9453c/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T08:29:47,653 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for a9b3cedf953393869a4d86b0d4e9453c 2024-11-24T08:29:47,654 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46165/hbase/data/default/testSequentialEditLogSeqNum/a9b3cedf953393869a4d86b0d4e9453c 2024-11-24T08:29:47,654 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46165/hbase/data/default/testSequentialEditLogSeqNum/a9b3cedf953393869a4d86b0d4e9453c 2024-11-24T08:29:47,655 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for a9b3cedf953393869a4d86b0d4e9453c 2024-11-24T08:29:47,655 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for a9b3cedf953393869a4d86b0d4e9453c 2024-11-24T08:29:47,657 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for a9b3cedf953393869a4d86b0d4e9453c 2024-11-24T08:29:47,659 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:46165/hbase/data/default/testSequentialEditLogSeqNum/a9b3cedf953393869a4d86b0d4e9453c/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-24T08:29:47,660 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened a9b3cedf953393869a4d86b0d4e9453c; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=70896229, jitterRate=0.05643613636493683}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-24T08:29:47,661 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for a9b3cedf953393869a4d86b0d4e9453c: Writing region info on filesystem at 1732436987637Initializing all the Stores at 1732436987650 (+13 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732436987650Cleaning up temporary data from old regions at 1732436987655 (+5 ms)Region opened successfully at 1732436987661 (+6 ms) 2024-11-24T08:29:47,675 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing a9b3cedf953393869a4d86b0d4e9453c 1/1 column families, dataSize=770 B heapSize=1.73 KB 2024-11-24T08:29:47,698 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46165/hbase/data/default/testSequentialEditLogSeqNum/a9b3cedf953393869a4d86b0d4e9453c/.tmp/a/b266d73fda8a4338a8d6fb2380464a7d is 81, key is testSequentialEditLogSeqNum/a:x0/1732436987661/Put/seqid=0 2024-11-24T08:29:47,704 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33555 is added to blk_1073741876_1053 (size=5833) 2024-11-24T08:29:47,705 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34459 is added to blk_1073741876_1053 (size=5833) 2024-11-24T08:29:47,706 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40533 is added to blk_1073741876_1053 (size=5833) 2024-11-24T08:29:47,706 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=770 B at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:46165/hbase/data/default/testSequentialEditLogSeqNum/a9b3cedf953393869a4d86b0d4e9453c/.tmp/a/b266d73fda8a4338a8d6fb2380464a7d 2024-11-24T08:29:47,714 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46165/hbase/data/default/testSequentialEditLogSeqNum/a9b3cedf953393869a4d86b0d4e9453c/.tmp/a/b266d73fda8a4338a8d6fb2380464a7d as hdfs://localhost:46165/hbase/data/default/testSequentialEditLogSeqNum/a9b3cedf953393869a4d86b0d4e9453c/a/b266d73fda8a4338a8d6fb2380464a7d 2024-11-24T08:29:47,763 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46165/hbase/data/default/testSequentialEditLogSeqNum/a9b3cedf953393869a4d86b0d4e9453c/a/b266d73fda8a4338a8d6fb2380464a7d, entries=10, sequenceid=13, filesize=5.7 K 2024-11-24T08:29:47,765 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~770 B/770, heapSize ~1.72 KB/1760, currentSize=0 B/0 for a9b3cedf953393869a4d86b0d4e9453c in 90ms, sequenceid=13, compaction requested=false 2024-11-24T08:29:47,765 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for a9b3cedf953393869a4d86b0d4e9453c: 2024-11-24T08:29:47,779 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T08:29:47,779 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T08:29:47,779 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T08:29:47,780 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T08:29:47,780 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T08:29:47,796 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33555 is added to blk_1073741874_1051 (size=1616) 2024-11-24T08:29:47,797 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34459 is added to blk_1073741874_1051 (size=1616) 2024-11-24T08:29:47,797 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40533 is added to blk_1073741874_1051 (size=1616) 2024-11-24T08:29:47,824 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:46165/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1732436987615/wal.1732436987624, size=1.6 K (1616bytes) 2024-11-24T08:29:47,825 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:46165/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1732436987615/wal.1732436987624 2024-11-24T08:29:47,825 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:46165/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1732436987615/wal.1732436987624 after 0ms 2024-11-24T08:29:47,829 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:46165/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1732436987615/wal.1732436987624: isRecoveredEdits=false, hasTagCompression=true, hasValueCompression=false, valueCompressionType=null 2024-11-24T08:29:47,829 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:46165/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1732436987615/wal.1732436987624 took 5ms 2024-11-24T08:29:47,832 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:46165/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1732436987615/wal.1732436987624 so closing down 2024-11-24T08:29:47,832 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-24T08:29:47,833 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000003-wal.1732436987624.temp 2024-11-24T08:29:47,835 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:46165/hbase/data/default/testSequentialEditLogSeqNum/a9b3cedf953393869a4d86b0d4e9453c/recovered.edits/0000000000000000003-wal.1732436987624.temp 2024-11-24T08:29:47,836 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-11-24T08:29:47,842 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33555 is added to blk_1073741877_1054 (size=1175) 2024-11-24T08:29:47,843 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40533 is added to blk_1073741877_1054 (size=1175) 2024-11-24T08:29:47,843 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34459 is added to blk_1073741877_1054 (size=1175) 2024-11-24T08:29:47,844 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:46165/hbase/data/default/testSequentialEditLogSeqNum/a9b3cedf953393869a4d86b0d4e9453c/recovered.edits/0000000000000000003-wal.1732436987624.temp (wrote 15 edits, skipped 0 edits in 0 ms) 2024-11-24T08:29:47,846 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:46165/hbase/data/default/testSequentialEditLogSeqNum/a9b3cedf953393869a4d86b0d4e9453c/recovered.edits/0000000000000000003-wal.1732436987624.temp to hdfs://localhost:46165/hbase/data/default/testSequentialEditLogSeqNum/a9b3cedf953393869a4d86b0d4e9453c/recovered.edits/0000000000000000020 2024-11-24T08:29:47,846 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 17 edits across 1 Regions in 16 ms; skipped=2; WAL=hdfs://localhost:46165/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1732436987615/wal.1732436987624, size=1.6 K, length=1616, corrupted=false, cancelled=false 2024-11-24T08:29:47,846 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:46165/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1732436987615/wal.1732436987624, journal: Splitting hdfs://localhost:46165/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1732436987615/wal.1732436987624, size=1.6 K (1616bytes) at 1732436987825Finishing writing output for hdfs://localhost:46165/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1732436987615/wal.1732436987624 so closing down at 1732436987832 (+7 ms)Creating recovered edits writer path=hdfs://localhost:46165/hbase/data/default/testSequentialEditLogSeqNum/a9b3cedf953393869a4d86b0d4e9453c/recovered.edits/0000000000000000003-wal.1732436987624.temp at 1732436987835 (+3 ms)3 split writer threads finished at 1732436987836 (+1 ms)Closed recovered edits writer path=hdfs://localhost:46165/hbase/data/default/testSequentialEditLogSeqNum/a9b3cedf953393869a4d86b0d4e9453c/recovered.edits/0000000000000000003-wal.1732436987624.temp (wrote 15 edits, skipped 0 edits in 0 ms) at 1732436987844 (+8 ms)Rename recovered edits hdfs://localhost:46165/hbase/data/default/testSequentialEditLogSeqNum/a9b3cedf953393869a4d86b0d4e9453c/recovered.edits/0000000000000000003-wal.1732436987624.temp to hdfs://localhost:46165/hbase/data/default/testSequentialEditLogSeqNum/a9b3cedf953393869a4d86b0d4e9453c/recovered.edits/0000000000000000020 at 1732436987846 (+2 ms)Processed 17 edits across 1 Regions in 16 ms; skipped=2; WAL=hdfs://localhost:46165/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1732436987615/wal.1732436987624, size=1.6 K, length=1616, corrupted=false, cancelled=false at 1732436987846 2024-11-24T08:29:47,864 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestAsyncWALReplayCompressed#testSequentialEditLogSeqNum Thread=407 (was 402) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1541013681_22 at /127.0.0.1:35226 [Waiting for operation #11] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1541013681_22 at /127.0.0.1:43844 [Waiting for operation #19] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: sync.2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441) Potentially hanging thread: sync.0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441) Potentially hanging thread: sync.4 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441) Potentially hanging thread: sync.3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441) Potentially hanging thread: sync.1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441) - Thread LEAK? -, OpenFileDescriptor=963 (was 927) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=153 (was 153), ProcessCount=11 (was 11), AvailableMemoryMB=6759 (was 6830) 2024-11-24T08:29:47,877 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestAsyncWALReplayCompressed#testRegionMadeOfBulkLoadedFilesOnly Thread=407, OpenFileDescriptor=963, MaxFileDescriptor=1048576, SystemLoadAverage=153, ProcessCount=11, AvailableMemoryMB=6758 2024-11-24T08:29:47,893 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-24T08:29:47,895 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-24T08:29:47,917 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-24T08:29:47,920 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-29254320, suffix=, logDir=hdfs://localhost:46165/hbase/WALs/hregion-29254320, archiveDir=hdfs://localhost:46165/hbase/oldWALs, maxLogs=32 2024-11-24T08:29:47,934 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-29254320/hregion-29254320.1732436987921, exclude list is [], retry=0 2024-11-24T08:29:47,938 DEBUG [AsyncFSWAL-17-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:34459,DS-6b4d1656-f49b-4a16-be49-2eb682c16e11,DISK] 2024-11-24T08:29:47,938 DEBUG [AsyncFSWAL-17-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33555,DS-8d97c3e2-1525-418c-a970-00a7157a0d31,DISK] 2024-11-24T08:29:47,939 DEBUG [AsyncFSWAL-17-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40533,DS-5eeb9666-6dd5-4a48-9fb1-3f9aff1acdca,DISK] 2024-11-24T08:29:47,944 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-29254320/hregion-29254320.1732436987921 2024-11-24T08:29:47,944 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:44131:44131),(127.0.0.1/127.0.0.1:39357:39357),(127.0.0.1/127.0.0.1:42673:42673)] 2024-11-24T08:29:47,945 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => a1afaa7558c1b8709569612f88fd0e39, NAME => 'testRegionMadeOfBulkLoadedFilesOnly,,1732436987893.a1afaa7558c1b8709569612f88fd0e39.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testRegionMadeOfBulkLoadedFilesOnly', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:46165/hbase 2024-11-24T08:29:47,962 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34459 is added to blk_1073741879_1056 (size=70) 2024-11-24T08:29:47,966 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33555 is added to blk_1073741879_1056 (size=70) 2024-11-24T08:29:47,966 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40533 is added to blk_1073741879_1056 (size=70) 2024-11-24T08:29:47,967 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testRegionMadeOfBulkLoadedFilesOnly,,1732436987893.a1afaa7558c1b8709569612f88fd0e39.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T08:29:47,972 INFO [StoreOpener-a1afaa7558c1b8709569612f88fd0e39-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region a1afaa7558c1b8709569612f88fd0e39 2024-11-24T08:29:47,974 INFO [StoreOpener-a1afaa7558c1b8709569612f88fd0e39-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region a1afaa7558c1b8709569612f88fd0e39 columnFamilyName a 2024-11-24T08:29:47,974 DEBUG [StoreOpener-a1afaa7558c1b8709569612f88fd0e39-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:29:47,975 INFO [StoreOpener-a1afaa7558c1b8709569612f88fd0e39-1 {}] regionserver.HStore(327): Store=a1afaa7558c1b8709569612f88fd0e39/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T08:29:47,975 INFO [StoreOpener-a1afaa7558c1b8709569612f88fd0e39-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region a1afaa7558c1b8709569612f88fd0e39 2024-11-24T08:29:47,976 INFO [StoreOpener-a1afaa7558c1b8709569612f88fd0e39-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region a1afaa7558c1b8709569612f88fd0e39 columnFamilyName b 2024-11-24T08:29:47,976 DEBUG [StoreOpener-a1afaa7558c1b8709569612f88fd0e39-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:29:47,977 INFO [StoreOpener-a1afaa7558c1b8709569612f88fd0e39-1 {}] regionserver.HStore(327): Store=a1afaa7558c1b8709569612f88fd0e39/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T08:29:47,977 INFO [StoreOpener-a1afaa7558c1b8709569612f88fd0e39-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region a1afaa7558c1b8709569612f88fd0e39 2024-11-24T08:29:47,978 INFO [StoreOpener-a1afaa7558c1b8709569612f88fd0e39-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region a1afaa7558c1b8709569612f88fd0e39 columnFamilyName c 2024-11-24T08:29:47,978 DEBUG [StoreOpener-a1afaa7558c1b8709569612f88fd0e39-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:29:47,979 INFO [StoreOpener-a1afaa7558c1b8709569612f88fd0e39-1 {}] regionserver.HStore(327): Store=a1afaa7558c1b8709569612f88fd0e39/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T08:29:47,979 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for a1afaa7558c1b8709569612f88fd0e39 2024-11-24T08:29:47,980 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46165/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/a1afaa7558c1b8709569612f88fd0e39 2024-11-24T08:29:47,980 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46165/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/a1afaa7558c1b8709569612f88fd0e39 2024-11-24T08:29:47,981 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for a1afaa7558c1b8709569612f88fd0e39 2024-11-24T08:29:47,981 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for a1afaa7558c1b8709569612f88fd0e39 2024-11-24T08:29:47,982 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testRegionMadeOfBulkLoadedFilesOnly descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-24T08:29:47,983 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for a1afaa7558c1b8709569612f88fd0e39 2024-11-24T08:29:47,986 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:46165/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/a1afaa7558c1b8709569612f88fd0e39/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-24T08:29:47,986 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened a1afaa7558c1b8709569612f88fd0e39; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=69282419, jitterRate=0.03238849341869354}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-24T08:29:47,987 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for a1afaa7558c1b8709569612f88fd0e39: Writing region info on filesystem at 1732436987967Initializing all the Stores at 1732436987968 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732436987968Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732436987972 (+4 ms)Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732436987972Cleaning up temporary data from old regions at 1732436987981 (+9 ms)Region opened successfully at 1732436987987 (+6 ms) 2024-11-24T08:29:47,987 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing a1afaa7558c1b8709569612f88fd0e39, disabling compactions & flushes 2024-11-24T08:29:47,987 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testRegionMadeOfBulkLoadedFilesOnly,,1732436987893.a1afaa7558c1b8709569612f88fd0e39. 2024-11-24T08:29:47,987 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testRegionMadeOfBulkLoadedFilesOnly,,1732436987893.a1afaa7558c1b8709569612f88fd0e39. 2024-11-24T08:29:47,987 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testRegionMadeOfBulkLoadedFilesOnly,,1732436987893.a1afaa7558c1b8709569612f88fd0e39. after waiting 0 ms 2024-11-24T08:29:47,987 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testRegionMadeOfBulkLoadedFilesOnly,,1732436987893.a1afaa7558c1b8709569612f88fd0e39. 2024-11-24T08:29:47,988 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testRegionMadeOfBulkLoadedFilesOnly,,1732436987893.a1afaa7558c1b8709569612f88fd0e39. 2024-11-24T08:29:47,989 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for a1afaa7558c1b8709569612f88fd0e39: Waiting for close lock at 1732436987987Disabling compacts and flushes for region at 1732436987987Disabling writes for close at 1732436987987Writing region close event to WAL at 1732436987988 (+1 ms)Closed at 1732436987988 2024-11-24T08:29:47,994 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40533 is added to blk_1073741878_1055 (size=93) 2024-11-24T08:29:47,994 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34459 is added to blk_1073741878_1055 (size=93) 2024-11-24T08:29:47,994 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33555 is added to blk_1073741878_1055 (size=93) 2024-11-24T08:29:47,997 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-11-24T08:29:47,997 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-29254320:(num 1732436987921) 2024-11-24T08:29:47,997 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-24T08:29:47,999 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:46165/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1732436987892, archiveDir=hdfs://localhost:46165/hbase/oldWALs, maxLogs=32 2024-11-24T08:29:48,021 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1732436987892/wal.1732436988000, exclude list is [], retry=0 2024-11-24T08:29:48,024 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33555,DS-8d97c3e2-1525-418c-a970-00a7157a0d31,DISK] 2024-11-24T08:29:48,025 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40533,DS-5eeb9666-6dd5-4a48-9fb1-3f9aff1acdca,DISK] 2024-11-24T08:29:48,025 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:34459,DS-6b4d1656-f49b-4a16-be49-2eb682c16e11,DISK] 2024-11-24T08:29:48,027 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1732436987892/wal.1732436988000 2024-11-24T08:29:48,028 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:39357:39357),(127.0.0.1/127.0.0.1:42673:42673),(127.0.0.1/127.0.0.1:44131:44131)] 2024-11-24T08:29:48,028 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => a1afaa7558c1b8709569612f88fd0e39, NAME => 'testRegionMadeOfBulkLoadedFilesOnly,,1732436987893.a1afaa7558c1b8709569612f88fd0e39.', STARTKEY => '', ENDKEY => ''} 2024-11-24T08:29:48,028 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testRegionMadeOfBulkLoadedFilesOnly,,1732436987893.a1afaa7558c1b8709569612f88fd0e39.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T08:29:48,028 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for a1afaa7558c1b8709569612f88fd0e39 2024-11-24T08:29:48,028 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for a1afaa7558c1b8709569612f88fd0e39 2024-11-24T08:29:48,032 INFO [StoreOpener-a1afaa7558c1b8709569612f88fd0e39-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region a1afaa7558c1b8709569612f88fd0e39 2024-11-24T08:29:48,034 INFO [StoreOpener-a1afaa7558c1b8709569612f88fd0e39-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region a1afaa7558c1b8709569612f88fd0e39 columnFamilyName a 2024-11-24T08:29:48,034 DEBUG [StoreOpener-a1afaa7558c1b8709569612f88fd0e39-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:29:48,035 INFO [StoreOpener-a1afaa7558c1b8709569612f88fd0e39-1 {}] regionserver.HStore(327): Store=a1afaa7558c1b8709569612f88fd0e39/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T08:29:48,035 INFO [StoreOpener-a1afaa7558c1b8709569612f88fd0e39-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region a1afaa7558c1b8709569612f88fd0e39 2024-11-24T08:29:48,036 INFO [StoreOpener-a1afaa7558c1b8709569612f88fd0e39-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region a1afaa7558c1b8709569612f88fd0e39 columnFamilyName b 2024-11-24T08:29:48,036 DEBUG [StoreOpener-a1afaa7558c1b8709569612f88fd0e39-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:29:48,037 INFO [StoreOpener-a1afaa7558c1b8709569612f88fd0e39-1 {}] regionserver.HStore(327): Store=a1afaa7558c1b8709569612f88fd0e39/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T08:29:48,037 INFO [StoreOpener-a1afaa7558c1b8709569612f88fd0e39-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region a1afaa7558c1b8709569612f88fd0e39 2024-11-24T08:29:48,038 INFO [StoreOpener-a1afaa7558c1b8709569612f88fd0e39-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region a1afaa7558c1b8709569612f88fd0e39 columnFamilyName c 2024-11-24T08:29:48,038 DEBUG [StoreOpener-a1afaa7558c1b8709569612f88fd0e39-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:29:48,038 INFO [StoreOpener-a1afaa7558c1b8709569612f88fd0e39-1 {}] regionserver.HStore(327): Store=a1afaa7558c1b8709569612f88fd0e39/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T08:29:48,038 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for a1afaa7558c1b8709569612f88fd0e39 2024-11-24T08:29:48,039 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46165/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/a1afaa7558c1b8709569612f88fd0e39 2024-11-24T08:29:48,040 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46165/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/a1afaa7558c1b8709569612f88fd0e39 2024-11-24T08:29:48,042 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for a1afaa7558c1b8709569612f88fd0e39 2024-11-24T08:29:48,042 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for a1afaa7558c1b8709569612f88fd0e39 2024-11-24T08:29:48,042 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testRegionMadeOfBulkLoadedFilesOnly descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-24T08:29:48,044 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for a1afaa7558c1b8709569612f88fd0e39 2024-11-24T08:29:48,044 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened a1afaa7558c1b8709569612f88fd0e39; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=71474512, jitterRate=0.06505322456359863}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-24T08:29:48,046 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for a1afaa7558c1b8709569612f88fd0e39: Writing region info on filesystem at 1732436988029Initializing all the Stores at 1732436988030 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732436988030Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732436988032 (+2 ms)Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732436988032Cleaning up temporary data from old regions at 1732436988042 (+10 ms)Region opened successfully at 1732436988045 (+3 ms) 2024-11-24T08:29:48,051 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46165/hbase/testRegionMadeOfBulkLoadedFilesOnly/hfile is 28, key is \x0D/a:a/1732436988049/Put/seqid=0 2024-11-24T08:29:48,057 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40533 is added to blk_1073741881_1058 (size=4826) 2024-11-24T08:29:48,058 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34459 is added to blk_1073741881_1058 (size=4826) 2024-11-24T08:29:48,060 INFO [Time-limited test {}] regionserver.HStore(614): Validating hfile at hdfs://localhost:46165/hbase/testRegionMadeOfBulkLoadedFilesOnly/hfile for inclusion in a1afaa7558c1b8709569612f88fd0e39/a 2024-11-24T08:29:48,060 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33555 is added to blk_1073741881_1058 (size=4826) 2024-11-24T08:29:48,070 DEBUG [Time-limited test {}] regionserver.HStore(626): HFile bounds: first= last=z 2024-11-24T08:29:48,070 DEBUG [Time-limited test {}] regionserver.HStore(628): Region bounds: first= last= 2024-11-24T08:29:48,070 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for a1afaa7558c1b8709569612f88fd0e39: 2024-11-24T08:29:48,072 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46165/hbase/testRegionMadeOfBulkLoadedFilesOnly/hfile as hdfs://localhost:46165/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/a1afaa7558c1b8709569612f88fd0e39/a/95f0f2e3e84b43a1abc75e3dc3394923_SeqId_3_ 2024-11-24T08:29:48,073 INFO [Time-limited test {}] regionserver.HStore(700): Loaded HFile hdfs://localhost:46165/hbase/testRegionMadeOfBulkLoadedFilesOnly/hfile into a1afaa7558c1b8709569612f88fd0e39/a as hdfs://localhost:46165/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/a1afaa7558c1b8709569612f88fd0e39/a/95f0f2e3e84b43a1abc75e3dc3394923_SeqId_3_ - updating store file list. 2024-11-24T08:29:48,079 DEBUG [Time-limited test {}] regionserver.HStoreFile(483): HFile Bloom filter type for 95f0f2e3e84b43a1abc75e3dc3394923_SeqId_3_: NONE, but ROW specified in column family configuration 2024-11-24T08:29:48,079 INFO [Time-limited test {}] regionserver.HStore(722): Loaded HFile hdfs://localhost:46165/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/a1afaa7558c1b8709569612f88fd0e39/a/95f0f2e3e84b43a1abc75e3dc3394923_SeqId_3_ into a1afaa7558c1b8709569612f88fd0e39/a 2024-11-24T08:29:48,079 INFO [Time-limited test {}] regionserver.HStore(706): Successfully loaded hdfs://localhost:46165/hbase/testRegionMadeOfBulkLoadedFilesOnly/hfile into a1afaa7558c1b8709569612f88fd0e39/a (new location: hdfs://localhost:46165/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/a1afaa7558c1b8709569612f88fd0e39/a/95f0f2e3e84b43a1abc75e3dc3394923_SeqId_3_) 2024-11-24T08:29:48,120 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:46165/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1732436987892/wal.1732436988000, size=0 (0bytes) 2024-11-24T08:29:48,120 WARN [Time-limited test {}] wal.WALSplitter(453): File hdfs://localhost:46165/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1732436987892/wal.1732436988000 might be still open, length is 0 2024-11-24T08:29:48,120 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:46165/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1732436987892/wal.1732436988000 2024-11-24T08:29:48,121 WARN [IPC Server handler 0 on default port 46165 {}] namenode.FSNamesystem(3854): DIR* NameSystem.internalReleaseLease: File /hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1732436987892/wal.1732436988000 has not been closed. Lease recovery is in progress. RecoveryId = 1059 for block blk_1073741880_1057 2024-11-24T08:29:48,121 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=0 on file=hdfs://localhost:46165/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1732436987892/wal.1732436988000 after 1ms 2024-11-24T08:29:50,749 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1541013681_22 at /127.0.0.1:44046 [Receiving block BP-1907186785-172.17.0.2-1732436973683:blk_1073741880_1057] {}] datanode.DataXceiver(331): 127.0.0.1:33555:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:44046 dst: /127.0.0.1:33555 java.io.InterruptedIOException: Interrupted while waiting for IO on channel java.nio.channels.SocketChannel[connected local=localhost/127.0.0.1:33555 remote=/127.0.0.1:44046]. Total timeout mills is 60000, 57337 millis timeout left. at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:350) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?] at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?] at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?] at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?] at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?] at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T08:29:50,749 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1541013681_22 at /127.0.0.1:39090 [Receiving block BP-1907186785-172.17.0.2-1732436973683:blk_1073741880_1057] {}] datanode.DataXceiver(331): 127.0.0.1:40533:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:39090 dst: /127.0.0.1:40533 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T08:29:50,749 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1541013681_22 at /127.0.0.1:35324 [Receiving block BP-1907186785-172.17.0.2-1732436973683:blk_1073741880_1057] {}] datanode.DataXceiver(331): 127.0.0.1:34459:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:35324 dst: /127.0.0.1:34459 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T08:29:50,753 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33555 is added to blk_1073741880_1059 (size=434) 2024-11-24T08:29:50,753 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40533 is added to blk_1073741880_1059 (size=434) 2024-11-24T08:29:50,754 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34459 is added to blk_1073741880_1059 (size=434) 2024-11-24T08:29:52,122 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=1 on file=hdfs://localhost:46165/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1732436987892/wal.1732436988000 after 4002ms 2024-11-24T08:29:52,125 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:46165/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1732436987892/wal.1732436988000: isRecoveredEdits=false, hasTagCompression=true, hasValueCompression=false, valueCompressionType=null 2024-11-24T08:29:52,126 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:46165/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1732436987892/wal.1732436988000 took 4006ms 2024-11-24T08:29:52,128 INFO [Time-limited test {}] wal.WALSplitter(493): EOF from hdfs://localhost:46165/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1732436987892/wal.1732436988000; continuing. 2024-11-24T08:29:52,128 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:46165/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1732436987892/wal.1732436988000 so closing down 2024-11-24T08:29:52,128 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-24T08:29:52,130 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000005-wal.1732436988000.temp 2024-11-24T08:29:52,131 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:46165/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/a1afaa7558c1b8709569612f88fd0e39/recovered.edits/0000000000000000005-wal.1732436988000.temp 2024-11-24T08:29:52,131 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-11-24T08:29:52,137 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33555 is added to blk_1073741882_1060 (size=236) 2024-11-24T08:29:52,137 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34459 is added to blk_1073741882_1060 (size=236) 2024-11-24T08:29:52,138 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40533 is added to blk_1073741882_1060 (size=236) 2024-11-24T08:29:52,139 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:46165/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/a1afaa7558c1b8709569612f88fd0e39/recovered.edits/0000000000000000005-wal.1732436988000.temp (wrote 1 edits, skipped 0 edits in 0 ms) 2024-11-24T08:29:52,140 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:46165/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/a1afaa7558c1b8709569612f88fd0e39/recovered.edits/0000000000000000005-wal.1732436988000.temp to hdfs://localhost:46165/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/a1afaa7558c1b8709569612f88fd0e39/recovered.edits/0000000000000000005 2024-11-24T08:29:52,140 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 2 edits across 1 Regions in 14 ms; skipped=1; WAL=hdfs://localhost:46165/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1732436987892/wal.1732436988000, size=0, length=0, corrupted=false, cancelled=false 2024-11-24T08:29:52,140 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:46165/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1732436987892/wal.1732436988000, journal: Splitting hdfs://localhost:46165/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1732436987892/wal.1732436988000, size=0 (0bytes) at 1732436988120Finishing writing output for hdfs://localhost:46165/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1732436987892/wal.1732436988000 so closing down at 1732436992128 (+4008 ms)Creating recovered edits writer path=hdfs://localhost:46165/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/a1afaa7558c1b8709569612f88fd0e39/recovered.edits/0000000000000000005-wal.1732436988000.temp at 1732436992131 (+3 ms)3 split writer threads finished at 1732436992131Closed recovered edits writer path=hdfs://localhost:46165/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/a1afaa7558c1b8709569612f88fd0e39/recovered.edits/0000000000000000005-wal.1732436988000.temp (wrote 1 edits, skipped 0 edits in 0 ms) at 1732436992139 (+8 ms)Rename recovered edits hdfs://localhost:46165/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/a1afaa7558c1b8709569612f88fd0e39/recovered.edits/0000000000000000005-wal.1732436988000.temp to hdfs://localhost:46165/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/a1afaa7558c1b8709569612f88fd0e39/recovered.edits/0000000000000000005 at 1732436992140 (+1 ms)Processed 2 edits across 1 Regions in 14 ms; skipped=1; WAL=hdfs://localhost:46165/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1732436987892/wal.1732436988000, size=0, length=0, corrupted=false, cancelled=false at 1732436992140 2024-11-24T08:29:52,142 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:46165/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1732436987892/wal.1732436988000 to hdfs://localhost:46165/hbase/oldWALs/wal.1732436988000 2024-11-24T08:29:52,143 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:46165/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/a1afaa7558c1b8709569612f88fd0e39/recovered.edits/0000000000000000005 2024-11-24T08:29:52,143 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-24T08:29:52,145 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:46165/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1732436987892, archiveDir=hdfs://localhost:46165/hbase/oldWALs, maxLogs=32 2024-11-24T08:29:52,160 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1732436987892/wal.1732436992146, exclude list is [], retry=0 2024-11-24T08:29:52,163 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:34459,DS-6b4d1656-f49b-4a16-be49-2eb682c16e11,DISK] 2024-11-24T08:29:52,163 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33555,DS-8d97c3e2-1525-418c-a970-00a7157a0d31,DISK] 2024-11-24T08:29:52,163 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40533,DS-5eeb9666-6dd5-4a48-9fb1-3f9aff1acdca,DISK] 2024-11-24T08:29:52,165 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1732436987892/wal.1732436992146 2024-11-24T08:29:52,166 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:44131:44131),(127.0.0.1/127.0.0.1:39357:39357),(127.0.0.1/127.0.0.1:42673:42673)] 2024-11-24T08:29:52,166 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => a1afaa7558c1b8709569612f88fd0e39, NAME => 'testRegionMadeOfBulkLoadedFilesOnly,,1732436987893.a1afaa7558c1b8709569612f88fd0e39.', STARTKEY => '', ENDKEY => ''} 2024-11-24T08:29:52,166 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testRegionMadeOfBulkLoadedFilesOnly,,1732436987893.a1afaa7558c1b8709569612f88fd0e39.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T08:29:52,166 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for a1afaa7558c1b8709569612f88fd0e39 2024-11-24T08:29:52,166 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for a1afaa7558c1b8709569612f88fd0e39 2024-11-24T08:29:52,168 INFO [StoreOpener-a1afaa7558c1b8709569612f88fd0e39-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region a1afaa7558c1b8709569612f88fd0e39 2024-11-24T08:29:52,169 INFO [StoreOpener-a1afaa7558c1b8709569612f88fd0e39-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region a1afaa7558c1b8709569612f88fd0e39 columnFamilyName a 2024-11-24T08:29:52,169 DEBUG [StoreOpener-a1afaa7558c1b8709569612f88fd0e39-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:29:52,175 DEBUG [StoreFileOpener-a1afaa7558c1b8709569612f88fd0e39-a-1 {}] regionserver.HStoreFile(483): HFile Bloom filter type for 95f0f2e3e84b43a1abc75e3dc3394923_SeqId_3_: NONE, but ROW specified in column family configuration 2024-11-24T08:29:52,175 DEBUG [StoreOpener-a1afaa7558c1b8709569612f88fd0e39-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:46165/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/a1afaa7558c1b8709569612f88fd0e39/a/95f0f2e3e84b43a1abc75e3dc3394923_SeqId_3_ 2024-11-24T08:29:52,175 INFO [StoreOpener-a1afaa7558c1b8709569612f88fd0e39-1 {}] regionserver.HStore(327): Store=a1afaa7558c1b8709569612f88fd0e39/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T08:29:52,175 INFO [StoreOpener-a1afaa7558c1b8709569612f88fd0e39-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region a1afaa7558c1b8709569612f88fd0e39 2024-11-24T08:29:52,176 INFO [StoreOpener-a1afaa7558c1b8709569612f88fd0e39-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region a1afaa7558c1b8709569612f88fd0e39 columnFamilyName b 2024-11-24T08:29:52,176 DEBUG [StoreOpener-a1afaa7558c1b8709569612f88fd0e39-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:29:52,177 INFO [StoreOpener-a1afaa7558c1b8709569612f88fd0e39-1 {}] regionserver.HStore(327): Store=a1afaa7558c1b8709569612f88fd0e39/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T08:29:52,177 INFO [StoreOpener-a1afaa7558c1b8709569612f88fd0e39-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region a1afaa7558c1b8709569612f88fd0e39 2024-11-24T08:29:52,178 INFO [StoreOpener-a1afaa7558c1b8709569612f88fd0e39-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region a1afaa7558c1b8709569612f88fd0e39 columnFamilyName c 2024-11-24T08:29:52,178 DEBUG [StoreOpener-a1afaa7558c1b8709569612f88fd0e39-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:29:52,178 INFO [StoreOpener-a1afaa7558c1b8709569612f88fd0e39-1 {}] regionserver.HStore(327): Store=a1afaa7558c1b8709569612f88fd0e39/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T08:29:52,178 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for a1afaa7558c1b8709569612f88fd0e39 2024-11-24T08:29:52,179 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46165/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/a1afaa7558c1b8709569612f88fd0e39 2024-11-24T08:29:52,180 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:46165/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/a1afaa7558c1b8709569612f88fd0e39 2024-11-24T08:29:52,181 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:46165/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/a1afaa7558c1b8709569612f88fd0e39/recovered.edits/0000000000000000005 2024-11-24T08:29:52,183 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:46165/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/a1afaa7558c1b8709569612f88fd0e39/recovered.edits/0000000000000000005: isRecoveredEdits=true, hasTagCompression=true, hasValueCompression=false, valueCompressionType=null 2024-11-24T08:29:52,184 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 1, skipped 0, firstSequenceIdInLog=5, maxSequenceIdInLog=5, path=hdfs://localhost:46165/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/a1afaa7558c1b8709569612f88fd0e39/recovered.edits/0000000000000000005 2024-11-24T08:29:52,184 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing a1afaa7558c1b8709569612f88fd0e39 3/3 column families, dataSize=58 B heapSize=904 B 2024-11-24T08:29:52,206 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46165/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/a1afaa7558c1b8709569612f88fd0e39/.tmp/a/7988ef2dd83841cc95202cd91f686980 is 62, key is testRegionMadeOfBulkLoadedFilesOnly/a:a/1732436988084/Put/seqid=0 2024-11-24T08:29:52,212 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34459 is added to blk_1073741884_1062 (size=5149) 2024-11-24T08:29:52,213 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33555 is added to blk_1073741884_1062 (size=5149) 2024-11-24T08:29:52,213 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40533 is added to blk_1073741884_1062 (size=5149) 2024-11-24T08:29:52,214 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=58 B at sequenceid=5 (bloomFilter=true), to=hdfs://localhost:46165/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/a1afaa7558c1b8709569612f88fd0e39/.tmp/a/7988ef2dd83841cc95202cd91f686980 2024-11-24T08:29:52,221 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46165/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/a1afaa7558c1b8709569612f88fd0e39/.tmp/a/7988ef2dd83841cc95202cd91f686980 as hdfs://localhost:46165/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/a1afaa7558c1b8709569612f88fd0e39/a/7988ef2dd83841cc95202cd91f686980 2024-11-24T08:29:52,228 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46165/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/a1afaa7558c1b8709569612f88fd0e39/a/7988ef2dd83841cc95202cd91f686980, entries=1, sequenceid=5, filesize=5.0 K 2024-11-24T08:29:52,228 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~58 B/58, heapSize ~376 B/376, currentSize=0 B/0 for a1afaa7558c1b8709569612f88fd0e39 in 44ms, sequenceid=5, compaction requested=false; wal=null 2024-11-24T08:29:52,229 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:46165/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/a1afaa7558c1b8709569612f88fd0e39/recovered.edits/0000000000000000005 2024-11-24T08:29:52,231 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for a1afaa7558c1b8709569612f88fd0e39 2024-11-24T08:29:52,231 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for a1afaa7558c1b8709569612f88fd0e39 2024-11-24T08:29:52,232 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testRegionMadeOfBulkLoadedFilesOnly descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-24T08:29:52,234 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for a1afaa7558c1b8709569612f88fd0e39 2024-11-24T08:29:52,237 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:46165/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/a1afaa7558c1b8709569612f88fd0e39/recovered.edits/5.seqid, newMaxSeqId=5, maxSeqId=1 2024-11-24T08:29:52,238 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened a1afaa7558c1b8709569612f88fd0e39; next sequenceid=6; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=70950897, jitterRate=0.05725075304508209}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-24T08:29:52,238 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for a1afaa7558c1b8709569612f88fd0e39: Writing region info on filesystem at 1732436992166Initializing all the Stores at 1732436992167 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732436992167Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732436992167Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732436992167Obtaining lock to block concurrent updates at 1732436992184 (+17 ms)Preparing flush snapshotting stores in a1afaa7558c1b8709569612f88fd0e39 at 1732436992184Finished memstore snapshotting testRegionMadeOfBulkLoadedFilesOnly,,1732436987893.a1afaa7558c1b8709569612f88fd0e39., syncing WAL and waiting on mvcc, flushsize=dataSize=58, getHeapSize=856, getOffHeapSize=0, getCellsCount=1 at 1732436992184Flushing stores of testRegionMadeOfBulkLoadedFilesOnly,,1732436987893.a1afaa7558c1b8709569612f88fd0e39. at 1732436992184Flushing a1afaa7558c1b8709569612f88fd0e39/a: creating writer at 1732436992184Flushing a1afaa7558c1b8709569612f88fd0e39/a: appending metadata at 1732436992205 (+21 ms)Flushing a1afaa7558c1b8709569612f88fd0e39/a: closing flushed file at 1732436992205Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@596b15b0: reopening flushed file at 1732436992220 (+15 ms)Finished flush of dataSize ~58 B/58, heapSize ~376 B/376, currentSize=0 B/0 for a1afaa7558c1b8709569612f88fd0e39 in 44ms, sequenceid=5, compaction requested=false; wal=null at 1732436992228 (+8 ms)Cleaning up temporary data from old regions at 1732436992231 (+3 ms)Region opened successfully at 1732436992238 (+7 ms) 2024-11-24T08:29:52,243 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing a1afaa7558c1b8709569612f88fd0e39, disabling compactions & flushes 2024-11-24T08:29:52,243 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testRegionMadeOfBulkLoadedFilesOnly,,1732436987893.a1afaa7558c1b8709569612f88fd0e39. 2024-11-24T08:29:52,243 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testRegionMadeOfBulkLoadedFilesOnly,,1732436987893.a1afaa7558c1b8709569612f88fd0e39. 2024-11-24T08:29:52,243 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testRegionMadeOfBulkLoadedFilesOnly,,1732436987893.a1afaa7558c1b8709569612f88fd0e39. after waiting 0 ms 2024-11-24T08:29:52,243 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testRegionMadeOfBulkLoadedFilesOnly,,1732436987893.a1afaa7558c1b8709569612f88fd0e39. 2024-11-24T08:29:52,244 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testRegionMadeOfBulkLoadedFilesOnly,,1732436987893.a1afaa7558c1b8709569612f88fd0e39. 2024-11-24T08:29:52,244 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for a1afaa7558c1b8709569612f88fd0e39: Waiting for close lock at 1732436992243Disabling compacts and flushes for region at 1732436992243Disabling writes for close at 1732436992243Writing region close event to WAL at 1732436992244 (+1 ms)Closed at 1732436992244 2024-11-24T08:29:52,247 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34459 is added to blk_1073741883_1061 (size=93) 2024-11-24T08:29:52,247 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40533 is added to blk_1073741883_1061 (size=93) 2024-11-24T08:29:52,248 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33555 is added to blk_1073741883_1061 (size=93) 2024-11-24T08:29:52,250 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-11-24T08:29:52,250 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL wal:(num 1732436992146) 2024-11-24T08:29:52,270 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestAsyncWALReplayCompressed#testRegionMadeOfBulkLoadedFilesOnly Thread=409 (was 407) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1462872822_22 at /127.0.0.1:58834 [Waiting for operation #5] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-17-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Client (1987708253) connection to localhost/127.0.0.1:46165 from jenkinstestRegionMadeOfBulkLoadedFilesOnly java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) Potentially hanging thread: AsyncFSWAL-17-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1462872822_22 at /127.0.0.1:47590 [Waiting for operation #5] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-17-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1462872822_22 at /127.0.0.1:53688 [Waiting for operation #7] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: LeaseRenewer:jenkinstestRegionMadeOfBulkLoadedFilesOnly@localhost:46165 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=1023 (was 963) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=149 (was 153), ProcessCount=11 (was 11), AvailableMemoryMB=6699 (was 6758) 2024-11-24T08:29:52,281 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestAsyncWALReplayCompressed#testReplayEditsAfterRegionMovedWithMultiCF Thread=409, OpenFileDescriptor=1023, MaxFileDescriptor=1048576, SystemLoadAverage=149, ProcessCount=11, AvailableMemoryMB=6698 2024-11-24T08:29:52,296 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-24T08:29:52,301 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-24T08:29:52,306 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] client.AsyncConnectionImpl(321): The fetched master address is 30c28c82771d,37719,1732436976835 2024-11-24T08:29:52,308 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@2c0fd56 2024-11-24T08:29:52,309 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-24T08:29:52,311 INFO [HMaster-EventLoopGroup-2-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50760, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-24T08:29:52,316 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37719 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'testReplayEditsAfterRegionMovedWithMultiCF', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'cf1', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'cf2', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-24T08:29:52,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37719 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=testReplayEditsAfterRegionMovedWithMultiCF 2024-11-24T08:29:52,327 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=testReplayEditsAfterRegionMovedWithMultiCF execute state=CREATE_TABLE_PRE_OPERATION 2024-11-24T08:29:52,329 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37719 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "testReplayEditsAfterRegionMovedWithMultiCF" procId is: 4 2024-11-24T08:29:52,330 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:29:52,332 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=testReplayEditsAfterRegionMovedWithMultiCF execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-24T08:29:52,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37719 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-24T08:29:52,344 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34459 is added to blk_1073741885_1063 (size=694) 2024-11-24T08:29:52,345 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33555 is added to blk_1073741885_1063 (size=694) 2024-11-24T08:29:52,345 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40533 is added to blk_1073741885_1063 (size=694) 2024-11-24T08:29:52,349 INFO [RegionOpenAndInit-testReplayEditsAfterRegionMovedWithMultiCF-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 25de72d74cf42cf025decee5a0d3cd6e, NAME => 'testReplayEditsAfterRegionMovedWithMultiCF,,1732436992313.25de72d74cf42cf025decee5a0d3cd6e.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testReplayEditsAfterRegionMovedWithMultiCF', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'cf1', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'cf2', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:46165/user/jenkins/test-data/7507fec3-e300-7585-3a43-9c9d3ed5b09e 2024-11-24T08:29:52,359 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33555 is added to blk_1073741886_1064 (size=77) 2024-11-24T08:29:52,360 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40533 is added to blk_1073741886_1064 (size=77) 2024-11-24T08:29:52,361 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34459 is added to blk_1073741886_1064 (size=77) 2024-11-24T08:29:52,361 DEBUG [RegionOpenAndInit-testReplayEditsAfterRegionMovedWithMultiCF-pool-0 {}] regionserver.HRegion(898): Instantiated testReplayEditsAfterRegionMovedWithMultiCF,,1732436992313.25de72d74cf42cf025decee5a0d3cd6e.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T08:29:52,361 DEBUG [RegionOpenAndInit-testReplayEditsAfterRegionMovedWithMultiCF-pool-0 {}] regionserver.HRegion(1722): Closing 25de72d74cf42cf025decee5a0d3cd6e, disabling compactions & flushes 2024-11-24T08:29:52,361 INFO [RegionOpenAndInit-testReplayEditsAfterRegionMovedWithMultiCF-pool-0 {}] regionserver.HRegion(1755): Closing region testReplayEditsAfterRegionMovedWithMultiCF,,1732436992313.25de72d74cf42cf025decee5a0d3cd6e. 2024-11-24T08:29:52,362 DEBUG [RegionOpenAndInit-testReplayEditsAfterRegionMovedWithMultiCF-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1732436992313.25de72d74cf42cf025decee5a0d3cd6e. 2024-11-24T08:29:52,362 DEBUG [RegionOpenAndInit-testReplayEditsAfterRegionMovedWithMultiCF-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1732436992313.25de72d74cf42cf025decee5a0d3cd6e. after waiting 0 ms 2024-11-24T08:29:52,362 DEBUG [RegionOpenAndInit-testReplayEditsAfterRegionMovedWithMultiCF-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsAfterRegionMovedWithMultiCF,,1732436992313.25de72d74cf42cf025decee5a0d3cd6e. 2024-11-24T08:29:52,362 INFO [RegionOpenAndInit-testReplayEditsAfterRegionMovedWithMultiCF-pool-0 {}] regionserver.HRegion(1973): Closed testReplayEditsAfterRegionMovedWithMultiCF,,1732436992313.25de72d74cf42cf025decee5a0d3cd6e. 2024-11-24T08:29:52,362 DEBUG [RegionOpenAndInit-testReplayEditsAfterRegionMovedWithMultiCF-pool-0 {}] regionserver.HRegion(1676): Region close journal for 25de72d74cf42cf025decee5a0d3cd6e: Waiting for close lock at 1732436992361Disabling compacts and flushes for region at 1732436992361Disabling writes for close at 1732436992362 (+1 ms)Writing region close event to WAL at 1732436992362Closed at 1732436992362 2024-11-24T08:29:52,364 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=testReplayEditsAfterRegionMovedWithMultiCF execute state=CREATE_TABLE_ADD_TO_META 2024-11-24T08:29:52,370 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testReplayEditsAfterRegionMovedWithMultiCF,,1732436992313.25de72d74cf42cf025decee5a0d3cd6e.","families":{"info":[{"qualifier":"regioninfo","vlen":76,"tag":[],"timestamp":"1732436992364"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732436992364"}]},"ts":"1732436992364"} 2024-11-24T08:29:52,373 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-24T08:29:52,375 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=testReplayEditsAfterRegionMovedWithMultiCF execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-24T08:29:52,379 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testReplayEditsAfterRegionMovedWithMultiCF","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732436992375"}]},"ts":"1732436992375"} 2024-11-24T08:29:52,383 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=testReplayEditsAfterRegionMovedWithMultiCF, state=ENABLING in hbase:meta 2024-11-24T08:29:52,384 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(204): Hosts are {30c28c82771d=0} racks are {/default-rack=0} 2024-11-24T08:29:52,385 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-11-24T08:29:52,385 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-11-24T08:29:52,385 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(303): server 2 has 0 regions 2024-11-24T08:29:52,385 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-11-24T08:29:52,385 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-11-24T08:29:52,385 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(310): server 2 is on host 0 2024-11-24T08:29:52,386 INFO [PEWorker-3 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-11-24T08:29:52,386 INFO [PEWorker-3 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-11-24T08:29:52,386 INFO [PEWorker-3 {}] balancer.BalancerClusterState(321): server 2 is on rack 0 2024-11-24T08:29:52,386 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-11-24T08:29:52,387 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=25de72d74cf42cf025decee5a0d3cd6e, ASSIGN}] 2024-11-24T08:29:52,390 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=25de72d74cf42cf025decee5a0d3cd6e, ASSIGN 2024-11-24T08:29:52,391 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=25de72d74cf42cf025decee5a0d3cd6e, ASSIGN; state=OFFLINE, location=30c28c82771d,38023,1732436977716; forceNewPlan=false, retain=false 2024-11-24T08:29:52,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37719 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-24T08:29:52,545 INFO [30c28c82771d:37719 {}] balancer.BaseLoadBalancer(388): Reassigned 1 regions. 1 retained the pre-restart assignment. 2024-11-24T08:29:52,546 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=25de72d74cf42cf025decee5a0d3cd6e, regionState=OPENING, regionLocation=30c28c82771d,38023,1732436977716 2024-11-24T08:29:52,550 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=25de72d74cf42cf025decee5a0d3cd6e, ASSIGN because future has completed 2024-11-24T08:29:52,552 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 25de72d74cf42cf025decee5a0d3cd6e, server=30c28c82771d,38023,1732436977716}] 2024-11-24T08:29:52,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37719 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-24T08:29:52,706 DEBUG [RSProcedureDispatcher-pool-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-24T08:29:52,708 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35459, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-24T08:29:52,714 INFO [RS_OPEN_REGION-regionserver/30c28c82771d:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(132): Open testReplayEditsAfterRegionMovedWithMultiCF,,1732436992313.25de72d74cf42cf025decee5a0d3cd6e. 2024-11-24T08:29:52,714 DEBUG [RS_OPEN_REGION-regionserver/30c28c82771d:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7752): Opening region: {ENCODED => 25de72d74cf42cf025decee5a0d3cd6e, NAME => 'testReplayEditsAfterRegionMovedWithMultiCF,,1732436992313.25de72d74cf42cf025decee5a0d3cd6e.', STARTKEY => '', ENDKEY => ''} 2024-11-24T08:29:52,714 DEBUG [RS_OPEN_REGION-regionserver/30c28c82771d:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testReplayEditsAfterRegionMovedWithMultiCF 25de72d74cf42cf025decee5a0d3cd6e 2024-11-24T08:29:52,714 DEBUG [RS_OPEN_REGION-regionserver/30c28c82771d:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(898): Instantiated testReplayEditsAfterRegionMovedWithMultiCF,,1732436992313.25de72d74cf42cf025decee5a0d3cd6e.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T08:29:52,714 DEBUG [RS_OPEN_REGION-regionserver/30c28c82771d:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7794): checking encryption for 25de72d74cf42cf025decee5a0d3cd6e 2024-11-24T08:29:52,715 DEBUG [RS_OPEN_REGION-regionserver/30c28c82771d:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7797): checking classloading for 25de72d74cf42cf025decee5a0d3cd6e 2024-11-24T08:29:52,716 INFO [StoreOpener-25de72d74cf42cf025decee5a0d3cd6e-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf1 of region 25de72d74cf42cf025decee5a0d3cd6e 2024-11-24T08:29:52,717 INFO [StoreOpener-25de72d74cf42cf025decee5a0d3cd6e-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 25de72d74cf42cf025decee5a0d3cd6e columnFamilyName cf1 2024-11-24T08:29:52,717 DEBUG [StoreOpener-25de72d74cf42cf025decee5a0d3cd6e-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:29:52,718 INFO [StoreOpener-25de72d74cf42cf025decee5a0d3cd6e-1 {}] regionserver.HStore(327): Store=25de72d74cf42cf025decee5a0d3cd6e/cf1, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T08:29:52,718 INFO [StoreOpener-25de72d74cf42cf025decee5a0d3cd6e-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf2 of region 25de72d74cf42cf025decee5a0d3cd6e 2024-11-24T08:29:52,720 INFO [StoreOpener-25de72d74cf42cf025decee5a0d3cd6e-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 25de72d74cf42cf025decee5a0d3cd6e columnFamilyName cf2 2024-11-24T08:29:52,720 DEBUG [StoreOpener-25de72d74cf42cf025decee5a0d3cd6e-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:29:52,720 INFO [StoreOpener-25de72d74cf42cf025decee5a0d3cd6e-1 {}] regionserver.HStore(327): Store=25de72d74cf42cf025decee5a0d3cd6e/cf2, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T08:29:52,721 DEBUG [RS_OPEN_REGION-regionserver/30c28c82771d:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1038): replaying wal for 25de72d74cf42cf025decee5a0d3cd6e 2024-11-24T08:29:52,721 DEBUG [RS_OPEN_REGION-regionserver/30c28c82771d:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46165/user/jenkins/test-data/7507fec3-e300-7585-3a43-9c9d3ed5b09e/data/default/testReplayEditsAfterRegionMovedWithMultiCF/25de72d74cf42cf025decee5a0d3cd6e 2024-11-24T08:29:52,722 DEBUG [RS_OPEN_REGION-regionserver/30c28c82771d:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46165/user/jenkins/test-data/7507fec3-e300-7585-3a43-9c9d3ed5b09e/data/default/testReplayEditsAfterRegionMovedWithMultiCF/25de72d74cf42cf025decee5a0d3cd6e 2024-11-24T08:29:52,723 DEBUG [RS_OPEN_REGION-regionserver/30c28c82771d:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1048): stopping wal replay for 25de72d74cf42cf025decee5a0d3cd6e 2024-11-24T08:29:52,723 DEBUG [RS_OPEN_REGION-regionserver/30c28c82771d:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1060): Cleaning up temporary data for 25de72d74cf42cf025decee5a0d3cd6e 2024-11-24T08:29:52,724 DEBUG [RS_OPEN_REGION-regionserver/30c28c82771d:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsAfterRegionMovedWithMultiCF descriptor;using region.getMemStoreFlushHeapSize/# of families (64.0 M)) instead. 2024-11-24T08:29:52,725 DEBUG [RS_OPEN_REGION-regionserver/30c28c82771d:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1093): writing seq id for 25de72d74cf42cf025decee5a0d3cd6e 2024-11-24T08:29:52,729 DEBUG [RS_OPEN_REGION-regionserver/30c28c82771d:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:46165/user/jenkins/test-data/7507fec3-e300-7585-3a43-9c9d3ed5b09e/data/default/testReplayEditsAfterRegionMovedWithMultiCF/25de72d74cf42cf025decee5a0d3cd6e/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-24T08:29:52,729 INFO [RS_OPEN_REGION-regionserver/30c28c82771d:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1114): Opened 25de72d74cf42cf025decee5a0d3cd6e; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=68674776, jitterRate=0.02333390712738037}}}, FlushLargeStoresPolicy{flushSizeLowerBound=67108864} 2024-11-24T08:29:52,730 DEBUG [RS_OPEN_REGION-regionserver/30c28c82771d:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 25de72d74cf42cf025decee5a0d3cd6e 2024-11-24T08:29:52,730 DEBUG [RS_OPEN_REGION-regionserver/30c28c82771d:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1006): Region open journal for 25de72d74cf42cf025decee5a0d3cd6e: Running coprocessor pre-open hook at 1732436992715Writing region info on filesystem at 1732436992715Initializing all the Stores at 1732436992716 (+1 ms)Instantiating store for column family {NAME => 'cf1', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732436992716Instantiating store for column family {NAME => 'cf2', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732436992716Cleaning up temporary data from old regions at 1732436992723 (+7 ms)Running coprocessor post-open hooks at 1732436992730 (+7 ms)Region opened successfully at 1732436992730 2024-11-24T08:29:52,732 INFO [RS_OPEN_REGION-regionserver/30c28c82771d:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2236): Post open deploy tasks for testReplayEditsAfterRegionMovedWithMultiCF,,1732436992313.25de72d74cf42cf025decee5a0d3cd6e., pid=6, masterSystemTime=1732436992706 2024-11-24T08:29:52,736 DEBUG [RS_OPEN_REGION-regionserver/30c28c82771d:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2266): Finished post open deploy task for testReplayEditsAfterRegionMovedWithMultiCF,,1732436992313.25de72d74cf42cf025decee5a0d3cd6e. 2024-11-24T08:29:52,736 INFO [RS_OPEN_REGION-regionserver/30c28c82771d:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(153): Opened testReplayEditsAfterRegionMovedWithMultiCF,,1732436992313.25de72d74cf42cf025decee5a0d3cd6e. 2024-11-24T08:29:52,737 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=25de72d74cf42cf025decee5a0d3cd6e, regionState=OPEN, openSeqNum=2, regionLocation=30c28c82771d,38023,1732436977716 2024-11-24T08:29:52,741 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 25de72d74cf42cf025decee5a0d3cd6e, server=30c28c82771d,38023,1732436977716 because future has completed 2024-11-24T08:29:52,747 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=6, resume processing ppid=5 2024-11-24T08:29:52,747 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure 25de72d74cf42cf025decee5a0d3cd6e, server=30c28c82771d,38023,1732436977716 in 191 msec 2024-11-24T08:29:52,750 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=5, resume processing ppid=4 2024-11-24T08:29:52,750 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=25de72d74cf42cf025decee5a0d3cd6e, ASSIGN in 360 msec 2024-11-24T08:29:52,751 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=testReplayEditsAfterRegionMovedWithMultiCF execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-24T08:29:52,752 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testReplayEditsAfterRegionMovedWithMultiCF","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732436992751"}]},"ts":"1732436992751"} 2024-11-24T08:29:52,754 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=testReplayEditsAfterRegionMovedWithMultiCF, state=ENABLED in hbase:meta 2024-11-24T08:29:52,756 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=testReplayEditsAfterRegionMovedWithMultiCF execute state=CREATE_TABLE_POST_OPERATION 2024-11-24T08:29:52,758 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=testReplayEditsAfterRegionMovedWithMultiCF in 438 msec 2024-11-24T08:29:52,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37719 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-24T08:29:52,966 INFO [RPCClient-NioEventLoopGroup-6-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:testReplayEditsAfterRegionMovedWithMultiCF completed 2024-11-24T08:29:52,966 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table testReplayEditsAfterRegionMovedWithMultiCF get assigned. Timeout = 60000ms 2024-11-24T08:29:52,967 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-24T08:29:52,973 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table testReplayEditsAfterRegionMovedWithMultiCF assigned to meta. Checking AM states. 2024-11-24T08:29:52,973 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-24T08:29:52,974 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table testReplayEditsAfterRegionMovedWithMultiCF assigned. 2024-11-24T08:29:52,989 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testReplayEditsAfterRegionMovedWithMultiCF', row='r1', locateType=CURRENT is [region=testReplayEditsAfterRegionMovedWithMultiCF,,1732436992313.25de72d74cf42cf025decee5a0d3cd6e., hostname=30c28c82771d,38023,1732436977716, seqNum=2] 2024-11-24T08:29:52,991 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-24T08:29:52,993 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39546, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-24T08:29:53,008 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37719 {}] master.HMaster(2410): Client=jenkins//172.17.0.2 move hri=25de72d74cf42cf025decee5a0d3cd6e, source=30c28c82771d,38023,1732436977716, destination=30c28c82771d,40117,1732436977557, warming up region on 30c28c82771d,40117,1732436977557 2024-11-24T08:29:53,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37719 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-24T08:29:53,011 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37719 {}] master.HMaster(2414): Client=jenkins//172.17.0.2 move hri=25de72d74cf42cf025decee5a0d3cd6e, source=30c28c82771d,38023,1732436977716, destination=30c28c82771d,40117,1732436977557, running balancer 2024-11-24T08:29:53,013 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53381, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-24T08:29:53,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37719 {}] procedure2.ProcedureExecutor(1139): Stored pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=25de72d74cf42cf025decee5a0d3cd6e, REOPEN/MOVE 2024-11-24T08:29:53,014 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=25de72d74cf42cf025decee5a0d3cd6e, REOPEN/MOVE 2024-11-24T08:29:53,016 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=25de72d74cf42cf025decee5a0d3cd6e, regionState=CLOSING, regionLocation=30c28c82771d,38023,1732436977716 2024-11-24T08:29:53,018 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40117 {}] regionserver.RSRpcServices(2066): Warmup testReplayEditsAfterRegionMovedWithMultiCF,,1732436992313.25de72d74cf42cf025decee5a0d3cd6e. 2024-11-24T08:29:53,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40117 {}] regionserver.HRegion(7855): Warmup {ENCODED => 25de72d74cf42cf025decee5a0d3cd6e, NAME => 'testReplayEditsAfterRegionMovedWithMultiCF,,1732436992313.25de72d74cf42cf025decee5a0d3cd6e.', STARTKEY => '', ENDKEY => ''} 2024-11-24T08:29:53,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40117 {}] regionserver.HRegion(898): Instantiated testReplayEditsAfterRegionMovedWithMultiCF,,1732436992313.25de72d74cf42cf025decee5a0d3cd6e.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T08:29:53,019 INFO [StoreOpener-25de72d74cf42cf025decee5a0d3cd6e-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf1 of region 25de72d74cf42cf025decee5a0d3cd6e 2024-11-24T08:29:53,019 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=25de72d74cf42cf025decee5a0d3cd6e, REOPEN/MOVE because future has completed 2024-11-24T08:29:53,020 DEBUG [PEWorker-3 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-24T08:29:53,020 INFO [StoreOpener-25de72d74cf42cf025decee5a0d3cd6e-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 25de72d74cf42cf025decee5a0d3cd6e columnFamilyName cf1 2024-11-24T08:29:53,020 DEBUG [StoreOpener-25de72d74cf42cf025decee5a0d3cd6e-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:29:53,020 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=8, ppid=7, state=RUNNABLE, hasLock=false; CloseRegionProcedure 25de72d74cf42cf025decee5a0d3cd6e, server=30c28c82771d,38023,1732436977716}] 2024-11-24T08:29:53,021 INFO [StoreOpener-25de72d74cf42cf025decee5a0d3cd6e-1 {}] regionserver.HStore(327): Store=25de72d74cf42cf025decee5a0d3cd6e/cf1, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T08:29:53,021 INFO [StoreOpener-25de72d74cf42cf025decee5a0d3cd6e-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf2 of region 25de72d74cf42cf025decee5a0d3cd6e 2024-11-24T08:29:53,022 INFO [StoreOpener-25de72d74cf42cf025decee5a0d3cd6e-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 25de72d74cf42cf025decee5a0d3cd6e columnFamilyName cf2 2024-11-24T08:29:53,022 DEBUG [StoreOpener-25de72d74cf42cf025decee5a0d3cd6e-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:29:53,022 INFO [StoreOpener-25de72d74cf42cf025decee5a0d3cd6e-1 {}] regionserver.HStore(327): Store=25de72d74cf42cf025decee5a0d3cd6e/cf2, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T08:29:53,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40117 {}] regionserver.HRegion(1722): Closing 25de72d74cf42cf025decee5a0d3cd6e, disabling compactions & flushes 2024-11-24T08:29:53,023 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40117 {}] regionserver.HRegion(1755): Closing region testReplayEditsAfterRegionMovedWithMultiCF,,1732436992313.25de72d74cf42cf025decee5a0d3cd6e. 2024-11-24T08:29:53,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40117 {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1732436992313.25de72d74cf42cf025decee5a0d3cd6e. 2024-11-24T08:29:53,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40117 {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1732436992313.25de72d74cf42cf025decee5a0d3cd6e. after waiting 0 ms 2024-11-24T08:29:53,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40117 {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsAfterRegionMovedWithMultiCF,,1732436992313.25de72d74cf42cf025decee5a0d3cd6e. 2024-11-24T08:29:53,023 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40117 {}] regionserver.HRegion(1973): Closed testReplayEditsAfterRegionMovedWithMultiCF,,1732436992313.25de72d74cf42cf025decee5a0d3cd6e. 2024-11-24T08:29:53,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40117 {}] regionserver.HRegion(1676): Region close journal for 25de72d74cf42cf025decee5a0d3cd6e: Waiting for close lock at 1732436993023Disabling compacts and flushes for region at 1732436993023Disabling writes for close at 1732436993023Writing region close event to WAL at 1732436993023Closed at 1732436993023 2024-11-24T08:29:53,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37719 {}] procedure.ProcedureSyncWait(219): waitFor pid=7 2024-11-24T08:29:53,181 INFO [RS_CLOSE_REGION-regionserver/30c28c82771d:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] handler.UnassignRegionHandler(122): Close 25de72d74cf42cf025decee5a0d3cd6e 2024-11-24T08:29:53,181 DEBUG [RS_CLOSE_REGION-regionserver/30c28c82771d:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-24T08:29:53,182 DEBUG [RS_CLOSE_REGION-regionserver/30c28c82771d:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1722): Closing 25de72d74cf42cf025decee5a0d3cd6e, disabling compactions & flushes 2024-11-24T08:29:53,182 INFO [RS_CLOSE_REGION-regionserver/30c28c82771d:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1755): Closing region testReplayEditsAfterRegionMovedWithMultiCF,,1732436992313.25de72d74cf42cf025decee5a0d3cd6e. 2024-11-24T08:29:53,182 DEBUG [RS_CLOSE_REGION-regionserver/30c28c82771d:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1732436992313.25de72d74cf42cf025decee5a0d3cd6e. 2024-11-24T08:29:53,182 DEBUG [RS_CLOSE_REGION-regionserver/30c28c82771d:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1732436992313.25de72d74cf42cf025decee5a0d3cd6e. after waiting 0 ms 2024-11-24T08:29:53,182 DEBUG [RS_CLOSE_REGION-regionserver/30c28c82771d:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsAfterRegionMovedWithMultiCF,,1732436992313.25de72d74cf42cf025decee5a0d3cd6e. 2024-11-24T08:29:53,182 INFO [RS_CLOSE_REGION-regionserver/30c28c82771d:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(2902): Flushing 25de72d74cf42cf025decee5a0d3cd6e 2/2 column families, dataSize=31 B heapSize=616 B 2024-11-24T08:29:53,199 DEBUG [RS_CLOSE_REGION-regionserver/30c28c82771d:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46165/user/jenkins/test-data/7507fec3-e300-7585-3a43-9c9d3ed5b09e/data/default/testReplayEditsAfterRegionMovedWithMultiCF/25de72d74cf42cf025decee5a0d3cd6e/.tmp/cf1/f0b82296da1a4c6ab7b2fd6e81027e61 is 35, key is r1/cf1:q/1732436992993/Put/seqid=0 2024-11-24T08:29:53,206 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40533 is added to blk_1073741887_1065 (size=4783) 2024-11-24T08:29:53,207 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34459 is added to blk_1073741887_1065 (size=4783) 2024-11-24T08:29:53,207 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33555 is added to blk_1073741887_1065 (size=4783) 2024-11-24T08:29:53,207 INFO [RS_CLOSE_REGION-regionserver/30c28c82771d:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31 B at sequenceid=5 (bloomFilter=false), to=hdfs://localhost:46165/user/jenkins/test-data/7507fec3-e300-7585-3a43-9c9d3ed5b09e/data/default/testReplayEditsAfterRegionMovedWithMultiCF/25de72d74cf42cf025decee5a0d3cd6e/.tmp/cf1/f0b82296da1a4c6ab7b2fd6e81027e61 2024-11-24T08:29:53,215 DEBUG [RS_CLOSE_REGION-regionserver/30c28c82771d:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46165/user/jenkins/test-data/7507fec3-e300-7585-3a43-9c9d3ed5b09e/data/default/testReplayEditsAfterRegionMovedWithMultiCF/25de72d74cf42cf025decee5a0d3cd6e/.tmp/cf1/f0b82296da1a4c6ab7b2fd6e81027e61 as hdfs://localhost:46165/user/jenkins/test-data/7507fec3-e300-7585-3a43-9c9d3ed5b09e/data/default/testReplayEditsAfterRegionMovedWithMultiCF/25de72d74cf42cf025decee5a0d3cd6e/cf1/f0b82296da1a4c6ab7b2fd6e81027e61 2024-11-24T08:29:53,221 INFO [RS_CLOSE_REGION-regionserver/30c28c82771d:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46165/user/jenkins/test-data/7507fec3-e300-7585-3a43-9c9d3ed5b09e/data/default/testReplayEditsAfterRegionMovedWithMultiCF/25de72d74cf42cf025decee5a0d3cd6e/cf1/f0b82296da1a4c6ab7b2fd6e81027e61, entries=1, sequenceid=5, filesize=4.7 K 2024-11-24T08:29:53,222 INFO [RS_CLOSE_REGION-regionserver/30c28c82771d:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(3140): Finished flush of dataSize ~31 B/31, heapSize ~344 B/344, currentSize=0 B/0 for 25de72d74cf42cf025decee5a0d3cd6e in 40ms, sequenceid=5, compaction requested=false 2024-11-24T08:29:53,222 DEBUG [RS_CLOSE_REGION-regionserver/30c28c82771d:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'testReplayEditsAfterRegionMovedWithMultiCF' 2024-11-24T08:29:53,228 DEBUG [RS_CLOSE_REGION-regionserver/30c28c82771d:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:46165/user/jenkins/test-data/7507fec3-e300-7585-3a43-9c9d3ed5b09e/data/default/testReplayEditsAfterRegionMovedWithMultiCF/25de72d74cf42cf025decee5a0d3cd6e/recovered.edits/8.seqid, newMaxSeqId=8, maxSeqId=1 2024-11-24T08:29:53,231 INFO [RS_CLOSE_REGION-regionserver/30c28c82771d:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1973): Closed testReplayEditsAfterRegionMovedWithMultiCF,,1732436992313.25de72d74cf42cf025decee5a0d3cd6e. 2024-11-24T08:29:53,231 DEBUG [RS_CLOSE_REGION-regionserver/30c28c82771d:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1676): Region close journal for 25de72d74cf42cf025decee5a0d3cd6e: Waiting for close lock at 1732436993182Running coprocessor pre-close hooks at 1732436993182Disabling compacts and flushes for region at 1732436993182Disabling writes for close at 1732436993182Obtaining lock to block concurrent updates at 1732436993182Preparing flush snapshotting stores in 25de72d74cf42cf025decee5a0d3cd6e at 1732436993182Finished memstore snapshotting testReplayEditsAfterRegionMovedWithMultiCF,,1732436992313.25de72d74cf42cf025decee5a0d3cd6e., syncing WAL and waiting on mvcc, flushsize=dataSize=31, getHeapSize=584, getOffHeapSize=0, getCellsCount=1 at 1732436993182Flushing stores of testReplayEditsAfterRegionMovedWithMultiCF,,1732436992313.25de72d74cf42cf025decee5a0d3cd6e. at 1732436993183 (+1 ms)Flushing 25de72d74cf42cf025decee5a0d3cd6e/cf1: creating writer at 1732436993183Flushing 25de72d74cf42cf025decee5a0d3cd6e/cf1: appending metadata at 1732436993199 (+16 ms)Flushing 25de72d74cf42cf025decee5a0d3cd6e/cf1: closing flushed file at 1732436993199Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@2a05a1e3: reopening flushed file at 1732436993214 (+15 ms)Finished flush of dataSize ~31 B/31, heapSize ~344 B/344, currentSize=0 B/0 for 25de72d74cf42cf025decee5a0d3cd6e in 40ms, sequenceid=5, compaction requested=false at 1732436993222 (+8 ms)Writing region close event to WAL at 1732436993223 (+1 ms)Running coprocessor post-close hooks at 1732436993228 (+5 ms)Closed at 1732436993231 (+3 ms) 2024-11-24T08:29:53,231 INFO [RS_CLOSE_REGION-regionserver/30c28c82771d:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegionServer(3302): Adding 25de72d74cf42cf025decee5a0d3cd6e move to 30c28c82771d,40117,1732436977557 record at close sequenceid=5 2024-11-24T08:29:53,235 INFO [RS_CLOSE_REGION-regionserver/30c28c82771d:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] handler.UnassignRegionHandler(157): Closed 25de72d74cf42cf025decee5a0d3cd6e 2024-11-24T08:29:53,236 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=25de72d74cf42cf025decee5a0d3cd6e, regionState=CLOSED 2024-11-24T08:29:53,239 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=8, ppid=7, state=RUNNABLE, hasLock=false; CloseRegionProcedure 25de72d74cf42cf025decee5a0d3cd6e, server=30c28c82771d,38023,1732436977716 because future has completed 2024-11-24T08:29:53,245 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=8, resume processing ppid=7 2024-11-24T08:29:53,245 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=8, ppid=7, state=SUCCESS, hasLock=false; CloseRegionProcedure 25de72d74cf42cf025decee5a0d3cd6e, server=30c28c82771d,38023,1732436977716 in 221 msec 2024-11-24T08:29:53,246 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=25de72d74cf42cf025decee5a0d3cd6e, REOPEN/MOVE; state=CLOSED, location=30c28c82771d,40117,1732436977557; forceNewPlan=false, retain=false 2024-11-24T08:29:53,397 INFO [30c28c82771d:37719 {}] balancer.BaseLoadBalancer(388): Reassigned 1 regions. 1 retained the pre-restart assignment. 2024-11-24T08:29:53,397 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=25de72d74cf42cf025decee5a0d3cd6e, regionState=OPENING, regionLocation=30c28c82771d,40117,1732436977557 2024-11-24T08:29:53,400 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=25de72d74cf42cf025decee5a0d3cd6e, REOPEN/MOVE because future has completed 2024-11-24T08:29:53,401 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=9, ppid=7, state=RUNNABLE, hasLock=false; OpenRegionProcedure 25de72d74cf42cf025decee5a0d3cd6e, server=30c28c82771d,40117,1732436977557}] 2024-11-24T08:29:53,559 INFO [RS_OPEN_REGION-regionserver/30c28c82771d:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] handler.AssignRegionHandler(132): Open testReplayEditsAfterRegionMovedWithMultiCF,,1732436992313.25de72d74cf42cf025decee5a0d3cd6e. 2024-11-24T08:29:53,559 DEBUG [RS_OPEN_REGION-regionserver/30c28c82771d:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(7752): Opening region: {ENCODED => 25de72d74cf42cf025decee5a0d3cd6e, NAME => 'testReplayEditsAfterRegionMovedWithMultiCF,,1732436992313.25de72d74cf42cf025decee5a0d3cd6e.', STARTKEY => '', ENDKEY => ''} 2024-11-24T08:29:53,560 DEBUG [RS_OPEN_REGION-regionserver/30c28c82771d:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testReplayEditsAfterRegionMovedWithMultiCF 25de72d74cf42cf025decee5a0d3cd6e 2024-11-24T08:29:53,560 DEBUG [RS_OPEN_REGION-regionserver/30c28c82771d:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(898): Instantiated testReplayEditsAfterRegionMovedWithMultiCF,,1732436992313.25de72d74cf42cf025decee5a0d3cd6e.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T08:29:53,560 DEBUG [RS_OPEN_REGION-regionserver/30c28c82771d:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(7794): checking encryption for 25de72d74cf42cf025decee5a0d3cd6e 2024-11-24T08:29:53,560 DEBUG [RS_OPEN_REGION-regionserver/30c28c82771d:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(7797): checking classloading for 25de72d74cf42cf025decee5a0d3cd6e 2024-11-24T08:29:53,562 INFO [StoreOpener-25de72d74cf42cf025decee5a0d3cd6e-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf1 of region 25de72d74cf42cf025decee5a0d3cd6e 2024-11-24T08:29:53,563 INFO [StoreOpener-25de72d74cf42cf025decee5a0d3cd6e-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 25de72d74cf42cf025decee5a0d3cd6e columnFamilyName cf1 2024-11-24T08:29:53,563 DEBUG [StoreOpener-25de72d74cf42cf025decee5a0d3cd6e-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:29:53,570 DEBUG [StoreOpener-25de72d74cf42cf025decee5a0d3cd6e-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:46165/user/jenkins/test-data/7507fec3-e300-7585-3a43-9c9d3ed5b09e/data/default/testReplayEditsAfterRegionMovedWithMultiCF/25de72d74cf42cf025decee5a0d3cd6e/cf1/f0b82296da1a4c6ab7b2fd6e81027e61 2024-11-24T08:29:53,570 INFO [StoreOpener-25de72d74cf42cf025decee5a0d3cd6e-1 {}] regionserver.HStore(327): Store=25de72d74cf42cf025decee5a0d3cd6e/cf1, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T08:29:53,570 INFO [StoreOpener-25de72d74cf42cf025decee5a0d3cd6e-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf2 of region 25de72d74cf42cf025decee5a0d3cd6e 2024-11-24T08:29:53,571 INFO [StoreOpener-25de72d74cf42cf025decee5a0d3cd6e-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 25de72d74cf42cf025decee5a0d3cd6e columnFamilyName cf2 2024-11-24T08:29:53,571 DEBUG [StoreOpener-25de72d74cf42cf025decee5a0d3cd6e-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:29:53,572 INFO [StoreOpener-25de72d74cf42cf025decee5a0d3cd6e-1 {}] regionserver.HStore(327): Store=25de72d74cf42cf025decee5a0d3cd6e/cf2, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T08:29:53,572 DEBUG [RS_OPEN_REGION-regionserver/30c28c82771d:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1038): replaying wal for 25de72d74cf42cf025decee5a0d3cd6e 2024-11-24T08:29:53,573 DEBUG [RS_OPEN_REGION-regionserver/30c28c82771d:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46165/user/jenkins/test-data/7507fec3-e300-7585-3a43-9c9d3ed5b09e/data/default/testReplayEditsAfterRegionMovedWithMultiCF/25de72d74cf42cf025decee5a0d3cd6e 2024-11-24T08:29:53,574 DEBUG [RS_OPEN_REGION-regionserver/30c28c82771d:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46165/user/jenkins/test-data/7507fec3-e300-7585-3a43-9c9d3ed5b09e/data/default/testReplayEditsAfterRegionMovedWithMultiCF/25de72d74cf42cf025decee5a0d3cd6e 2024-11-24T08:29:53,575 DEBUG [RS_OPEN_REGION-regionserver/30c28c82771d:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1048): stopping wal replay for 25de72d74cf42cf025decee5a0d3cd6e 2024-11-24T08:29:53,575 DEBUG [RS_OPEN_REGION-regionserver/30c28c82771d:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1060): Cleaning up temporary data for 25de72d74cf42cf025decee5a0d3cd6e 2024-11-24T08:29:53,576 DEBUG [RS_OPEN_REGION-regionserver/30c28c82771d:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsAfterRegionMovedWithMultiCF descriptor;using region.getMemStoreFlushHeapSize/# of families (64.0 M)) instead. 2024-11-24T08:29:53,577 DEBUG [RS_OPEN_REGION-regionserver/30c28c82771d:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1093): writing seq id for 25de72d74cf42cf025decee5a0d3cd6e 2024-11-24T08:29:53,578 INFO [RS_OPEN_REGION-regionserver/30c28c82771d:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1114): Opened 25de72d74cf42cf025decee5a0d3cd6e; next sequenceid=9; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=68524730, jitterRate=0.021098047494888306}}}, FlushLargeStoresPolicy{flushSizeLowerBound=67108864} 2024-11-24T08:29:53,578 DEBUG [RS_OPEN_REGION-regionserver/30c28c82771d:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 25de72d74cf42cf025decee5a0d3cd6e 2024-11-24T08:29:53,579 DEBUG [RS_OPEN_REGION-regionserver/30c28c82771d:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1006): Region open journal for 25de72d74cf42cf025decee5a0d3cd6e: Running coprocessor pre-open hook at 1732436993560Writing region info on filesystem at 1732436993560Initializing all the Stores at 1732436993561 (+1 ms)Instantiating store for column family {NAME => 'cf1', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732436993561Instantiating store for column family {NAME => 'cf2', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732436993561Cleaning up temporary data from old regions at 1732436993575 (+14 ms)Running coprocessor post-open hooks at 1732436993578 (+3 ms)Region opened successfully at 1732436993579 (+1 ms) 2024-11-24T08:29:53,581 INFO [RS_OPEN_REGION-regionserver/30c28c82771d:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegionServer(2236): Post open deploy tasks for testReplayEditsAfterRegionMovedWithMultiCF,,1732436992313.25de72d74cf42cf025decee5a0d3cd6e., pid=9, masterSystemTime=1732436993554 2024-11-24T08:29:53,583 DEBUG [RS_OPEN_REGION-regionserver/30c28c82771d:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegionServer(2266): Finished post open deploy task for testReplayEditsAfterRegionMovedWithMultiCF,,1732436992313.25de72d74cf42cf025decee5a0d3cd6e. 2024-11-24T08:29:53,583 INFO [RS_OPEN_REGION-regionserver/30c28c82771d:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] handler.AssignRegionHandler(153): Opened testReplayEditsAfterRegionMovedWithMultiCF,,1732436992313.25de72d74cf42cf025decee5a0d3cd6e. 2024-11-24T08:29:53,584 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=25de72d74cf42cf025decee5a0d3cd6e, regionState=OPEN, openSeqNum=9, regionLocation=30c28c82771d,40117,1732436977557 2024-11-24T08:29:53,587 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=9, ppid=7, state=RUNNABLE, hasLock=false; OpenRegionProcedure 25de72d74cf42cf025decee5a0d3cd6e, server=30c28c82771d,40117,1732436977557 because future has completed 2024-11-24T08:29:53,590 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=37719 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=30c28c82771d,40117,1732436977557, table=testReplayEditsAfterRegionMovedWithMultiCF, region=25de72d74cf42cf025decee5a0d3cd6e. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-24T08:29:53,593 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=9, resume processing ppid=7 2024-11-24T08:29:53,593 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=9, ppid=7, state=SUCCESS, hasLock=false; OpenRegionProcedure 25de72d74cf42cf025decee5a0d3cd6e, server=30c28c82771d,40117,1732436977557 in 189 msec 2024-11-24T08:29:53,595 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=7, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=25de72d74cf42cf025decee5a0d3cd6e, REOPEN/MOVE in 581 msec 2024-11-24T08:29:53,620 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-24T08:29:53,622 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46002, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-24T08:29:53,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38023 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 139 connection: 172.17.0.2:39546 deadline: 1732437053627, exception=org.apache.hadoop.hbase.exceptions.RegionMovedException: Region moved to: hostname=30c28c82771d port=40117 startCode=1732436977557. As of locationSeqNum=5. 2024-11-24T08:29:53,651 DEBUG [RPCClient-NioEventLoopGroup-6-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1732436992313.25de72d74cf42cf025decee5a0d3cd6e., hostname=30c28c82771d,38023,1732436977716, seqNum=2 , the old value is region=testReplayEditsAfterRegionMovedWithMultiCF,,1732436992313.25de72d74cf42cf025decee5a0d3cd6e., hostname=30c28c82771d,38023,1732436977716, seqNum=2, error=org.apache.hadoop.hbase.exceptions.RegionMovedException: Region moved to: hostname=30c28c82771d port=40117 startCode=1732436977557. As of locationSeqNum=5. 2024-11-24T08:29:53,651 DEBUG [RPCClient-NioEventLoopGroup-6-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1732436992313.25de72d74cf42cf025decee5a0d3cd6e., hostname=30c28c82771d,38023,1732436977716, seqNum=2 is org.apache.hadoop.hbase.exceptions.RegionMovedException: Region moved to: hostname=30c28c82771d port=40117 startCode=1732436977557. As of locationSeqNum=5. 2024-11-24T08:29:53,651 DEBUG [RPCClient-NioEventLoopGroup-6-5 {}] client.AsyncRegionLocatorHelper(84): Try updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1732436992313.25de72d74cf42cf025decee5a0d3cd6e., hostname=30c28c82771d,38023,1732436977716, seqNum=2 with the new location region=testReplayEditsAfterRegionMovedWithMultiCF,,1732436992313.25de72d74cf42cf025decee5a0d3cd6e., hostname=30c28c82771d,40117,1732436977557, seqNum=5 constructed by org.apache.hadoop.hbase.exceptions.RegionMovedException: Region moved to: hostname=30c28c82771d port=40117 startCode=1732436977557. As of locationSeqNum=5. 2024-11-24T08:29:53,764 DEBUG [Async-Client-Retry-Timer-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-24T08:29:53,767 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46008, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-24T08:29:53,777 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 25de72d74cf42cf025decee5a0d3cd6e 2/2 column families, dataSize=50 B heapSize=720 B 2024-11-24T08:29:53,795 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46165/user/jenkins/test-data/7507fec3-e300-7585-3a43-9c9d3ed5b09e/data/default/testReplayEditsAfterRegionMovedWithMultiCF/25de72d74cf42cf025decee5a0d3cd6e/.tmp/cf1/5938977f2d074ce0b389d7d4f86d73b2 is 29, key is r1/cf1:/1732436993768/DeleteFamily/seqid=0 2024-11-24T08:29:53,803 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40533 is added to blk_1073741888_1066 (size=4906) 2024-11-24T08:29:53,803 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33555 is added to blk_1073741888_1066 (size=4906) 2024-11-24T08:29:53,803 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34459 is added to blk_1073741888_1066 (size=4906) 2024-11-24T08:29:53,804 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=25 B at sequenceid=12 (bloomFilter=false), to=hdfs://localhost:46165/user/jenkins/test-data/7507fec3-e300-7585-3a43-9c9d3ed5b09e/data/default/testReplayEditsAfterRegionMovedWithMultiCF/25de72d74cf42cf025decee5a0d3cd6e/.tmp/cf1/5938977f2d074ce0b389d7d4f86d73b2 2024-11-24T08:29:53,811 INFO [Time-limited test {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 5938977f2d074ce0b389d7d4f86d73b2 2024-11-24T08:29:53,827 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46165/user/jenkins/test-data/7507fec3-e300-7585-3a43-9c9d3ed5b09e/data/default/testReplayEditsAfterRegionMovedWithMultiCF/25de72d74cf42cf025decee5a0d3cd6e/.tmp/cf2/0a6dde3c069d4c619150c1d3d74ec2cb is 29, key is r1/cf2:/1732436993768/DeleteFamily/seqid=0 2024-11-24T08:29:53,835 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40533 is added to blk_1073741889_1067 (size=4906) 2024-11-24T08:29:53,835 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33555 is added to blk_1073741889_1067 (size=4906) 2024-11-24T08:29:53,835 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34459 is added to blk_1073741889_1067 (size=4906) 2024-11-24T08:29:53,836 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=25 B at sequenceid=12 (bloomFilter=false), to=hdfs://localhost:46165/user/jenkins/test-data/7507fec3-e300-7585-3a43-9c9d3ed5b09e/data/default/testReplayEditsAfterRegionMovedWithMultiCF/25de72d74cf42cf025decee5a0d3cd6e/.tmp/cf2/0a6dde3c069d4c619150c1d3d74ec2cb 2024-11-24T08:29:53,842 INFO [Time-limited test {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 0a6dde3c069d4c619150c1d3d74ec2cb 2024-11-24T08:29:53,843 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46165/user/jenkins/test-data/7507fec3-e300-7585-3a43-9c9d3ed5b09e/data/default/testReplayEditsAfterRegionMovedWithMultiCF/25de72d74cf42cf025decee5a0d3cd6e/.tmp/cf1/5938977f2d074ce0b389d7d4f86d73b2 as hdfs://localhost:46165/user/jenkins/test-data/7507fec3-e300-7585-3a43-9c9d3ed5b09e/data/default/testReplayEditsAfterRegionMovedWithMultiCF/25de72d74cf42cf025decee5a0d3cd6e/cf1/5938977f2d074ce0b389d7d4f86d73b2 2024-11-24T08:29:53,850 INFO [Time-limited test {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 5938977f2d074ce0b389d7d4f86d73b2 2024-11-24T08:29:53,850 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46165/user/jenkins/test-data/7507fec3-e300-7585-3a43-9c9d3ed5b09e/data/default/testReplayEditsAfterRegionMovedWithMultiCF/25de72d74cf42cf025decee5a0d3cd6e/cf1/5938977f2d074ce0b389d7d4f86d73b2, entries=1, sequenceid=12, filesize=4.8 K 2024-11-24T08:29:53,851 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46165/user/jenkins/test-data/7507fec3-e300-7585-3a43-9c9d3ed5b09e/data/default/testReplayEditsAfterRegionMovedWithMultiCF/25de72d74cf42cf025decee5a0d3cd6e/.tmp/cf2/0a6dde3c069d4c619150c1d3d74ec2cb as hdfs://localhost:46165/user/jenkins/test-data/7507fec3-e300-7585-3a43-9c9d3ed5b09e/data/default/testReplayEditsAfterRegionMovedWithMultiCF/25de72d74cf42cf025decee5a0d3cd6e/cf2/0a6dde3c069d4c619150c1d3d74ec2cb 2024-11-24T08:29:53,859 INFO [Time-limited test {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 0a6dde3c069d4c619150c1d3d74ec2cb 2024-11-24T08:29:53,859 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46165/user/jenkins/test-data/7507fec3-e300-7585-3a43-9c9d3ed5b09e/data/default/testReplayEditsAfterRegionMovedWithMultiCF/25de72d74cf42cf025decee5a0d3cd6e/cf2/0a6dde3c069d4c619150c1d3d74ec2cb, entries=1, sequenceid=12, filesize=4.8 K 2024-11-24T08:29:53,861 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~50 B/50, heapSize ~688 B/688, currentSize=0 B/0 for 25de72d74cf42cf025decee5a0d3cd6e in 84ms, sequenceid=12, compaction requested=false 2024-11-24T08:29:53,861 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for 25de72d74cf42cf025decee5a0d3cd6e: 2024-11-24T08:29:53,865 DEBUG [Time-limited test {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 2 store files, 0 compacting, 2 eligible, 16 blocking 2024-11-24T08:29:53,867 DEBUG [Time-limited test {}] regionserver.HStore(1541): 25de72d74cf42cf025decee5a0d3cd6e/cf1 is initiating major compaction (all files) 2024-11-24T08:29:53,867 INFO [Time-limited test {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-24T08:29:53,867 INFO [Time-limited test {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-24T08:29:53,867 INFO [Time-limited test {}] regionserver.HRegion(2416): Starting compaction of 25de72d74cf42cf025decee5a0d3cd6e/cf1 in testReplayEditsAfterRegionMovedWithMultiCF,,1732436992313.25de72d74cf42cf025decee5a0d3cd6e. 2024-11-24T08:29:53,868 INFO [Time-limited test {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46165/user/jenkins/test-data/7507fec3-e300-7585-3a43-9c9d3ed5b09e/data/default/testReplayEditsAfterRegionMovedWithMultiCF/25de72d74cf42cf025decee5a0d3cd6e/cf1/f0b82296da1a4c6ab7b2fd6e81027e61, hdfs://localhost:46165/user/jenkins/test-data/7507fec3-e300-7585-3a43-9c9d3ed5b09e/data/default/testReplayEditsAfterRegionMovedWithMultiCF/25de72d74cf42cf025decee5a0d3cd6e/cf1/5938977f2d074ce0b389d7d4f86d73b2] into tmpdir=hdfs://localhost:46165/user/jenkins/test-data/7507fec3-e300-7585-3a43-9c9d3ed5b09e/data/default/testReplayEditsAfterRegionMovedWithMultiCF/25de72d74cf42cf025decee5a0d3cd6e/.tmp, totalSize=9.5 K 2024-11-24T08:29:53,869 DEBUG [Time-limited test {}] compactions.Compactor(225): Compacting f0b82296da1a4c6ab7b2fd6e81027e61, keycount=1, bloomtype=NONE, size=4.7 K, encoding=NONE, compression=NONE, seqNum=5, earliestPutTs=1732436992993 2024-11-24T08:29:53,870 DEBUG [Time-limited test {}] compactions.Compactor(225): Compacting 5938977f2d074ce0b389d7d4f86d73b2, keycount=1, bloomtype=NONE, size=4.8 K, encoding=NONE, compression=NONE, seqNum=12, earliestPutTs=9223372036854775807 2024-11-24T08:29:53,882 INFO [Time-limited test {}] throttle.PressureAwareThroughputController(145): 25de72d74cf42cf025decee5a0d3cd6e#cf1#compaction#16 average throughput is NaN MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-24T08:29:53,889 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34459 is added to blk_1073741890_1068 (size=4626) 2024-11-24T08:29:53,889 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33555 is added to blk_1073741890_1068 (size=4626) 2024-11-24T08:29:53,889 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40533 is added to blk_1073741890_1068 (size=4626) 2024-11-24T08:29:53,899 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46165/user/jenkins/test-data/7507fec3-e300-7585-3a43-9c9d3ed5b09e/data/default/testReplayEditsAfterRegionMovedWithMultiCF/25de72d74cf42cf025decee5a0d3cd6e/.tmp/cf1/60c7344f8e564ddf862b0c7395425cee as hdfs://localhost:46165/user/jenkins/test-data/7507fec3-e300-7585-3a43-9c9d3ed5b09e/data/default/testReplayEditsAfterRegionMovedWithMultiCF/25de72d74cf42cf025decee5a0d3cd6e/cf1/60c7344f8e564ddf862b0c7395425cee 2024-11-24T08:29:53,915 INFO [Time-limited test {}] regionserver.HStore(1337): Completed major compaction of 2 (all) file(s) in 25de72d74cf42cf025decee5a0d3cd6e/cf1 of 25de72d74cf42cf025decee5a0d3cd6e into 60c7344f8e564ddf862b0c7395425cee(size=4.5 K), total size for store is 4.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T08:29:53,915 DEBUG [Time-limited test {}] regionserver.HRegion(2446): Compaction status journal for 25de72d74cf42cf025decee5a0d3cd6e: 2024-11-24T08:29:53,915 DEBUG [Time-limited test {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 1 store files, 0 compacting, 1 eligible, 16 blocking 2024-11-24T08:29:53,915 DEBUG [Time-limited test {}] regionserver.HStore(1541): 25de72d74cf42cf025decee5a0d3cd6e/cf2 is initiating major compaction (all files) 2024-11-24T08:29:53,915 INFO [Time-limited test {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-24T08:29:53,915 INFO [Time-limited test {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-24T08:29:53,915 INFO [Time-limited test {}] regionserver.HRegion(2416): Starting compaction of 25de72d74cf42cf025decee5a0d3cd6e/cf2 in testReplayEditsAfterRegionMovedWithMultiCF,,1732436992313.25de72d74cf42cf025decee5a0d3cd6e. 2024-11-24T08:29:53,915 INFO [Time-limited test {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46165/user/jenkins/test-data/7507fec3-e300-7585-3a43-9c9d3ed5b09e/data/default/testReplayEditsAfterRegionMovedWithMultiCF/25de72d74cf42cf025decee5a0d3cd6e/cf2/0a6dde3c069d4c619150c1d3d74ec2cb] into tmpdir=hdfs://localhost:46165/user/jenkins/test-data/7507fec3-e300-7585-3a43-9c9d3ed5b09e/data/default/testReplayEditsAfterRegionMovedWithMultiCF/25de72d74cf42cf025decee5a0d3cd6e/.tmp, totalSize=4.8 K 2024-11-24T08:29:53,916 DEBUG [Time-limited test {}] compactions.Compactor(225): Compacting 0a6dde3c069d4c619150c1d3d74ec2cb, keycount=1, bloomtype=NONE, size=4.8 K, encoding=NONE, compression=NONE, seqNum=12, earliestPutTs=9223372036854775807 2024-11-24T08:29:53,923 INFO [Time-limited test {}] throttle.PressureAwareThroughputController(145): 25de72d74cf42cf025decee5a0d3cd6e#cf2#compaction#17 average throughput is NaN MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-24T08:29:53,934 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34459 is added to blk_1073741891_1069 (size=4592) 2024-11-24T08:29:53,935 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33555 is added to blk_1073741891_1069 (size=4592) 2024-11-24T08:29:53,935 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40533 is added to blk_1073741891_1069 (size=4592) 2024-11-24T08:29:53,944 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46165/user/jenkins/test-data/7507fec3-e300-7585-3a43-9c9d3ed5b09e/data/default/testReplayEditsAfterRegionMovedWithMultiCF/25de72d74cf42cf025decee5a0d3cd6e/.tmp/cf2/84a6767619e642f88ded3e4c594f638d as hdfs://localhost:46165/user/jenkins/test-data/7507fec3-e300-7585-3a43-9c9d3ed5b09e/data/default/testReplayEditsAfterRegionMovedWithMultiCF/25de72d74cf42cf025decee5a0d3cd6e/cf2/84a6767619e642f88ded3e4c594f638d 2024-11-24T08:29:53,953 INFO [Time-limited test {}] regionserver.HStore(1337): Completed major compaction of 1 (all) file(s) in 25de72d74cf42cf025decee5a0d3cd6e/cf2 of 25de72d74cf42cf025decee5a0d3cd6e into 84a6767619e642f88ded3e4c594f638d(size=4.5 K), total size for store is 4.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T08:29:53,953 DEBUG [Time-limited test {}] regionserver.HRegion(2446): Compaction status journal for 25de72d74cf42cf025decee5a0d3cd6e: 2024-11-24T08:29:53,959 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37719 {}] master.HMaster(2410): Client=jenkins//172.17.0.2 move hri=25de72d74cf42cf025decee5a0d3cd6e, source=30c28c82771d,40117,1732436977557, destination=30c28c82771d,38023,1732436977716, warming up region on 30c28c82771d,38023,1732436977716 2024-11-24T08:29:53,959 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37719 {}] master.HMaster(2414): Client=jenkins//172.17.0.2 move hri=25de72d74cf42cf025decee5a0d3cd6e, source=30c28c82771d,40117,1732436977557, destination=30c28c82771d,38023,1732436977716, running balancer 2024-11-24T08:29:53,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37719 {}] procedure2.ProcedureExecutor(1139): Stored pid=10, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=25de72d74cf42cf025decee5a0d3cd6e, REOPEN/MOVE 2024-11-24T08:29:53,961 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=10, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=25de72d74cf42cf025decee5a0d3cd6e, REOPEN/MOVE 2024-11-24T08:29:53,963 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38023 {}] regionserver.RSRpcServices(2066): Warmup testReplayEditsAfterRegionMovedWithMultiCF,,1732436992313.25de72d74cf42cf025decee5a0d3cd6e. 2024-11-24T08:29:53,963 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=10 updating hbase:meta row=25de72d74cf42cf025decee5a0d3cd6e, regionState=CLOSING, regionLocation=30c28c82771d,40117,1732436977557 2024-11-24T08:29:53,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38023 {}] regionserver.HRegion(7855): Warmup {ENCODED => 25de72d74cf42cf025decee5a0d3cd6e, NAME => 'testReplayEditsAfterRegionMovedWithMultiCF,,1732436992313.25de72d74cf42cf025decee5a0d3cd6e.', STARTKEY => '', ENDKEY => ''} 2024-11-24T08:29:53,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38023 {}] regionserver.HRegion(898): Instantiated testReplayEditsAfterRegionMovedWithMultiCF,,1732436992313.25de72d74cf42cf025decee5a0d3cd6e.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T08:29:53,964 INFO [StoreOpener-25de72d74cf42cf025decee5a0d3cd6e-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf1 of region 25de72d74cf42cf025decee5a0d3cd6e 2024-11-24T08:29:53,966 INFO [StoreOpener-25de72d74cf42cf025decee5a0d3cd6e-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 25de72d74cf42cf025decee5a0d3cd6e columnFamilyName cf1 2024-11-24T08:29:53,966 DEBUG [StoreOpener-25de72d74cf42cf025decee5a0d3cd6e-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:29:53,967 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=10, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=25de72d74cf42cf025decee5a0d3cd6e, REOPEN/MOVE because future has completed 2024-11-24T08:29:53,967 DEBUG [PEWorker-5 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-24T08:29:53,967 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=11, ppid=10, state=RUNNABLE, hasLock=false; CloseRegionProcedure 25de72d74cf42cf025decee5a0d3cd6e, server=30c28c82771d,40117,1732436977557}] 2024-11-24T08:29:53,977 INFO [StoreFileOpener-25de72d74cf42cf025decee5a0d3cd6e-cf1-1 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 5938977f2d074ce0b389d7d4f86d73b2 2024-11-24T08:29:53,978 DEBUG [StoreOpener-25de72d74cf42cf025decee5a0d3cd6e-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:46165/user/jenkins/test-data/7507fec3-e300-7585-3a43-9c9d3ed5b09e/data/default/testReplayEditsAfterRegionMovedWithMultiCF/25de72d74cf42cf025decee5a0d3cd6e/cf1/5938977f2d074ce0b389d7d4f86d73b2 2024-11-24T08:29:53,982 DEBUG [StoreOpener-25de72d74cf42cf025decee5a0d3cd6e-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:46165/user/jenkins/test-data/7507fec3-e300-7585-3a43-9c9d3ed5b09e/data/default/testReplayEditsAfterRegionMovedWithMultiCF/25de72d74cf42cf025decee5a0d3cd6e/cf1/60c7344f8e564ddf862b0c7395425cee 2024-11-24T08:29:53,987 DEBUG [StoreOpener-25de72d74cf42cf025decee5a0d3cd6e-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:46165/user/jenkins/test-data/7507fec3-e300-7585-3a43-9c9d3ed5b09e/data/default/testReplayEditsAfterRegionMovedWithMultiCF/25de72d74cf42cf025decee5a0d3cd6e/cf1/f0b82296da1a4c6ab7b2fd6e81027e61 2024-11-24T08:29:53,987 INFO [StoreOpener-25de72d74cf42cf025decee5a0d3cd6e-1 {}] regionserver.HStore(327): Store=25de72d74cf42cf025decee5a0d3cd6e/cf1, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T08:29:53,988 INFO [StoreOpener-25de72d74cf42cf025decee5a0d3cd6e-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf2 of region 25de72d74cf42cf025decee5a0d3cd6e 2024-11-24T08:29:53,989 INFO [StoreOpener-25de72d74cf42cf025decee5a0d3cd6e-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 25de72d74cf42cf025decee5a0d3cd6e columnFamilyName cf2 2024-11-24T08:29:53,989 DEBUG [StoreOpener-25de72d74cf42cf025decee5a0d3cd6e-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:29:53,994 INFO [StoreFileOpener-25de72d74cf42cf025decee5a0d3cd6e-cf2-1 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 0a6dde3c069d4c619150c1d3d74ec2cb 2024-11-24T08:29:53,994 DEBUG [StoreOpener-25de72d74cf42cf025decee5a0d3cd6e-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:46165/user/jenkins/test-data/7507fec3-e300-7585-3a43-9c9d3ed5b09e/data/default/testReplayEditsAfterRegionMovedWithMultiCF/25de72d74cf42cf025decee5a0d3cd6e/cf2/0a6dde3c069d4c619150c1d3d74ec2cb 2024-11-24T08:29:54,000 DEBUG [StoreOpener-25de72d74cf42cf025decee5a0d3cd6e-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:46165/user/jenkins/test-data/7507fec3-e300-7585-3a43-9c9d3ed5b09e/data/default/testReplayEditsAfterRegionMovedWithMultiCF/25de72d74cf42cf025decee5a0d3cd6e/cf2/84a6767619e642f88ded3e4c594f638d 2024-11-24T08:29:54,000 INFO [StoreOpener-25de72d74cf42cf025decee5a0d3cd6e-1 {}] regionserver.HStore(327): Store=25de72d74cf42cf025decee5a0d3cd6e/cf2, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T08:29:54,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38023 {}] regionserver.HRegion(1722): Closing 25de72d74cf42cf025decee5a0d3cd6e, disabling compactions & flushes 2024-11-24T08:29:54,000 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38023 {}] regionserver.HRegion(1755): Closing region testReplayEditsAfterRegionMovedWithMultiCF,,1732436992313.25de72d74cf42cf025decee5a0d3cd6e. 2024-11-24T08:29:54,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38023 {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1732436992313.25de72d74cf42cf025decee5a0d3cd6e. 2024-11-24T08:29:54,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38023 {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1732436992313.25de72d74cf42cf025decee5a0d3cd6e. after waiting 0 ms 2024-11-24T08:29:54,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38023 {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsAfterRegionMovedWithMultiCF,,1732436992313.25de72d74cf42cf025decee5a0d3cd6e. 2024-11-24T08:29:54,002 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38023 {}] regionserver.HRegion(1973): Closed testReplayEditsAfterRegionMovedWithMultiCF,,1732436992313.25de72d74cf42cf025decee5a0d3cd6e. 2024-11-24T08:29:54,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38023 {}] regionserver.HRegion(1676): Region close journal for 25de72d74cf42cf025decee5a0d3cd6e: Waiting for close lock at 1732436994000Disabling compacts and flushes for region at 1732436994000Disabling writes for close at 1732436994000Writing region close event to WAL at 1732436994002 (+2 ms)Closed at 1732436994002 2024-11-24T08:29:54,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37719 {}] procedure.ProcedureSyncWait(219): waitFor pid=10 2024-11-24T08:29:54,122 INFO [RS_CLOSE_REGION-regionserver/30c28c82771d:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] handler.UnassignRegionHandler(122): Close 25de72d74cf42cf025decee5a0d3cd6e 2024-11-24T08:29:54,122 DEBUG [RS_CLOSE_REGION-regionserver/30c28c82771d:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-24T08:29:54,122 DEBUG [RS_CLOSE_REGION-regionserver/30c28c82771d:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] regionserver.HRegion(1722): Closing 25de72d74cf42cf025decee5a0d3cd6e, disabling compactions & flushes 2024-11-24T08:29:54,122 INFO [RS_CLOSE_REGION-regionserver/30c28c82771d:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] regionserver.HRegion(1755): Closing region testReplayEditsAfterRegionMovedWithMultiCF,,1732436992313.25de72d74cf42cf025decee5a0d3cd6e. 2024-11-24T08:29:54,122 DEBUG [RS_CLOSE_REGION-regionserver/30c28c82771d:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1732436992313.25de72d74cf42cf025decee5a0d3cd6e. 2024-11-24T08:29:54,122 DEBUG [RS_CLOSE_REGION-regionserver/30c28c82771d:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1732436992313.25de72d74cf42cf025decee5a0d3cd6e. after waiting 0 ms 2024-11-24T08:29:54,122 DEBUG [RS_CLOSE_REGION-regionserver/30c28c82771d:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsAfterRegionMovedWithMultiCF,,1732436992313.25de72d74cf42cf025decee5a0d3cd6e. 2024-11-24T08:29:54,123 DEBUG [StoreCloser-testReplayEditsAfterRegionMovedWithMultiCF,,1732436992313.25de72d74cf42cf025decee5a0d3cd6e.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:46165/user/jenkins/test-data/7507fec3-e300-7585-3a43-9c9d3ed5b09e/data/default/testReplayEditsAfterRegionMovedWithMultiCF/25de72d74cf42cf025decee5a0d3cd6e/cf1/f0b82296da1a4c6ab7b2fd6e81027e61, hdfs://localhost:46165/user/jenkins/test-data/7507fec3-e300-7585-3a43-9c9d3ed5b09e/data/default/testReplayEditsAfterRegionMovedWithMultiCF/25de72d74cf42cf025decee5a0d3cd6e/cf1/5938977f2d074ce0b389d7d4f86d73b2] to archive 2024-11-24T08:29:54,126 DEBUG [StoreCloser-testReplayEditsAfterRegionMovedWithMultiCF,,1732436992313.25de72d74cf42cf025decee5a0d3cd6e.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-24T08:29:54,129 DEBUG [StoreCloser-testReplayEditsAfterRegionMovedWithMultiCF,,1732436992313.25de72d74cf42cf025decee5a0d3cd6e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46165/user/jenkins/test-data/7507fec3-e300-7585-3a43-9c9d3ed5b09e/data/default/testReplayEditsAfterRegionMovedWithMultiCF/25de72d74cf42cf025decee5a0d3cd6e/cf1/f0b82296da1a4c6ab7b2fd6e81027e61 to hdfs://localhost:46165/user/jenkins/test-data/7507fec3-e300-7585-3a43-9c9d3ed5b09e/archive/data/default/testReplayEditsAfterRegionMovedWithMultiCF/25de72d74cf42cf025decee5a0d3cd6e/cf1/f0b82296da1a4c6ab7b2fd6e81027e61 2024-11-24T08:29:54,131 DEBUG [StoreCloser-testReplayEditsAfterRegionMovedWithMultiCF,,1732436992313.25de72d74cf42cf025decee5a0d3cd6e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46165/user/jenkins/test-data/7507fec3-e300-7585-3a43-9c9d3ed5b09e/data/default/testReplayEditsAfterRegionMovedWithMultiCF/25de72d74cf42cf025decee5a0d3cd6e/cf1/5938977f2d074ce0b389d7d4f86d73b2 to hdfs://localhost:46165/user/jenkins/test-data/7507fec3-e300-7585-3a43-9c9d3ed5b09e/archive/data/default/testReplayEditsAfterRegionMovedWithMultiCF/25de72d74cf42cf025decee5a0d3cd6e/cf1/5938977f2d074ce0b389d7d4f86d73b2 2024-11-24T08:29:54,145 DEBUG [StoreCloser-testReplayEditsAfterRegionMovedWithMultiCF,,1732436992313.25de72d74cf42cf025decee5a0d3cd6e.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:46165/user/jenkins/test-data/7507fec3-e300-7585-3a43-9c9d3ed5b09e/data/default/testReplayEditsAfterRegionMovedWithMultiCF/25de72d74cf42cf025decee5a0d3cd6e/cf2/0a6dde3c069d4c619150c1d3d74ec2cb] to archive 2024-11-24T08:29:54,145 DEBUG [StoreCloser-testReplayEditsAfterRegionMovedWithMultiCF,,1732436992313.25de72d74cf42cf025decee5a0d3cd6e.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-24T08:29:54,148 DEBUG [StoreCloser-testReplayEditsAfterRegionMovedWithMultiCF,,1732436992313.25de72d74cf42cf025decee5a0d3cd6e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46165/user/jenkins/test-data/7507fec3-e300-7585-3a43-9c9d3ed5b09e/data/default/testReplayEditsAfterRegionMovedWithMultiCF/25de72d74cf42cf025decee5a0d3cd6e/cf2/0a6dde3c069d4c619150c1d3d74ec2cb to hdfs://localhost:46165/user/jenkins/test-data/7507fec3-e300-7585-3a43-9c9d3ed5b09e/archive/data/default/testReplayEditsAfterRegionMovedWithMultiCF/25de72d74cf42cf025decee5a0d3cd6e/cf2/0a6dde3c069d4c619150c1d3d74ec2cb 2024-11-24T08:29:54,153 DEBUG [RS_CLOSE_REGION-regionserver/30c28c82771d:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:46165/user/jenkins/test-data/7507fec3-e300-7585-3a43-9c9d3ed5b09e/data/default/testReplayEditsAfterRegionMovedWithMultiCF/25de72d74cf42cf025decee5a0d3cd6e/recovered.edits/17.seqid, newMaxSeqId=17, maxSeqId=8 2024-11-24T08:29:54,153 INFO [RS_CLOSE_REGION-regionserver/30c28c82771d:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] regionserver.HRegion(1973): Closed testReplayEditsAfterRegionMovedWithMultiCF,,1732436992313.25de72d74cf42cf025decee5a0d3cd6e. 2024-11-24T08:29:54,154 DEBUG [RS_CLOSE_REGION-regionserver/30c28c82771d:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] regionserver.HRegion(1676): Region close journal for 25de72d74cf42cf025decee5a0d3cd6e: Waiting for close lock at 1732436994122Running coprocessor pre-close hooks at 1732436994122Disabling compacts and flushes for region at 1732436994122Disabling writes for close at 1732436994122Writing region close event to WAL at 1732436994149 (+27 ms)Running coprocessor post-close hooks at 1732436994153 (+4 ms)Closed at 1732436994153 2024-11-24T08:29:54,154 INFO [RS_CLOSE_REGION-regionserver/30c28c82771d:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] regionserver.HRegionServer(3302): Adding 25de72d74cf42cf025decee5a0d3cd6e move to 30c28c82771d,38023,1732436977716 record at close sequenceid=12 2024-11-24T08:29:54,156 INFO [RS_CLOSE_REGION-regionserver/30c28c82771d:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] handler.UnassignRegionHandler(157): Closed 25de72d74cf42cf025decee5a0d3cd6e 2024-11-24T08:29:54,157 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=10 updating hbase:meta row=25de72d74cf42cf025decee5a0d3cd6e, regionState=CLOSED 2024-11-24T08:29:54,159 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=11, ppid=10, state=RUNNABLE, hasLock=false; CloseRegionProcedure 25de72d74cf42cf025decee5a0d3cd6e, server=30c28c82771d,40117,1732436977557 because future has completed 2024-11-24T08:29:54,164 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=11, resume processing ppid=10 2024-11-24T08:29:54,164 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=11, ppid=10, state=SUCCESS, hasLock=false; CloseRegionProcedure 25de72d74cf42cf025decee5a0d3cd6e, server=30c28c82771d,40117,1732436977557 in 194 msec 2024-11-24T08:29:54,165 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=10, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=25de72d74cf42cf025decee5a0d3cd6e, REOPEN/MOVE; state=CLOSED, location=30c28c82771d,38023,1732436977716; forceNewPlan=false, retain=false 2024-11-24T08:29:54,315 INFO [30c28c82771d:37719 {}] balancer.BaseLoadBalancer(388): Reassigned 1 regions. 1 retained the pre-restart assignment. 2024-11-24T08:29:54,316 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=10 updating hbase:meta row=25de72d74cf42cf025decee5a0d3cd6e, regionState=OPENING, regionLocation=30c28c82771d,38023,1732436977716 2024-11-24T08:29:54,319 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=10, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=25de72d74cf42cf025decee5a0d3cd6e, REOPEN/MOVE because future has completed 2024-11-24T08:29:54,319 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=12, ppid=10, state=RUNNABLE, hasLock=false; OpenRegionProcedure 25de72d74cf42cf025decee5a0d3cd6e, server=30c28c82771d,38023,1732436977716}] 2024-11-24T08:29:54,477 INFO [RS_OPEN_REGION-regionserver/30c28c82771d:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] handler.AssignRegionHandler(132): Open testReplayEditsAfterRegionMovedWithMultiCF,,1732436992313.25de72d74cf42cf025decee5a0d3cd6e. 2024-11-24T08:29:54,477 DEBUG [RS_OPEN_REGION-regionserver/30c28c82771d:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(7752): Opening region: {ENCODED => 25de72d74cf42cf025decee5a0d3cd6e, NAME => 'testReplayEditsAfterRegionMovedWithMultiCF,,1732436992313.25de72d74cf42cf025decee5a0d3cd6e.', STARTKEY => '', ENDKEY => ''} 2024-11-24T08:29:54,477 DEBUG [RS_OPEN_REGION-regionserver/30c28c82771d:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testReplayEditsAfterRegionMovedWithMultiCF 25de72d74cf42cf025decee5a0d3cd6e 2024-11-24T08:29:54,477 DEBUG [RS_OPEN_REGION-regionserver/30c28c82771d:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(898): Instantiated testReplayEditsAfterRegionMovedWithMultiCF,,1732436992313.25de72d74cf42cf025decee5a0d3cd6e.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T08:29:54,478 DEBUG [RS_OPEN_REGION-regionserver/30c28c82771d:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(7794): checking encryption for 25de72d74cf42cf025decee5a0d3cd6e 2024-11-24T08:29:54,478 DEBUG [RS_OPEN_REGION-regionserver/30c28c82771d:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(7797): checking classloading for 25de72d74cf42cf025decee5a0d3cd6e 2024-11-24T08:29:54,479 INFO [StoreOpener-25de72d74cf42cf025decee5a0d3cd6e-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf1 of region 25de72d74cf42cf025decee5a0d3cd6e 2024-11-24T08:29:54,480 INFO [StoreOpener-25de72d74cf42cf025decee5a0d3cd6e-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 25de72d74cf42cf025decee5a0d3cd6e columnFamilyName cf1 2024-11-24T08:29:54,480 DEBUG [StoreOpener-25de72d74cf42cf025decee5a0d3cd6e-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:29:54,487 DEBUG [StoreOpener-25de72d74cf42cf025decee5a0d3cd6e-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:46165/user/jenkins/test-data/7507fec3-e300-7585-3a43-9c9d3ed5b09e/data/default/testReplayEditsAfterRegionMovedWithMultiCF/25de72d74cf42cf025decee5a0d3cd6e/cf1/60c7344f8e564ddf862b0c7395425cee 2024-11-24T08:29:54,487 INFO [StoreOpener-25de72d74cf42cf025decee5a0d3cd6e-1 {}] regionserver.HStore(327): Store=25de72d74cf42cf025decee5a0d3cd6e/cf1, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T08:29:54,487 INFO [StoreOpener-25de72d74cf42cf025decee5a0d3cd6e-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf2 of region 25de72d74cf42cf025decee5a0d3cd6e 2024-11-24T08:29:54,488 INFO [StoreOpener-25de72d74cf42cf025decee5a0d3cd6e-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 25de72d74cf42cf025decee5a0d3cd6e columnFamilyName cf2 2024-11-24T08:29:54,488 DEBUG [StoreOpener-25de72d74cf42cf025decee5a0d3cd6e-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:29:54,501 DEBUG [StoreOpener-25de72d74cf42cf025decee5a0d3cd6e-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:46165/user/jenkins/test-data/7507fec3-e300-7585-3a43-9c9d3ed5b09e/data/default/testReplayEditsAfterRegionMovedWithMultiCF/25de72d74cf42cf025decee5a0d3cd6e/cf2/84a6767619e642f88ded3e4c594f638d 2024-11-24T08:29:54,501 INFO [StoreOpener-25de72d74cf42cf025decee5a0d3cd6e-1 {}] regionserver.HStore(327): Store=25de72d74cf42cf025decee5a0d3cd6e/cf2, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T08:29:54,501 DEBUG [RS_OPEN_REGION-regionserver/30c28c82771d:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1038): replaying wal for 25de72d74cf42cf025decee5a0d3cd6e 2024-11-24T08:29:54,502 DEBUG [RS_OPEN_REGION-regionserver/30c28c82771d:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46165/user/jenkins/test-data/7507fec3-e300-7585-3a43-9c9d3ed5b09e/data/default/testReplayEditsAfterRegionMovedWithMultiCF/25de72d74cf42cf025decee5a0d3cd6e 2024-11-24T08:29:54,504 DEBUG [RS_OPEN_REGION-regionserver/30c28c82771d:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46165/user/jenkins/test-data/7507fec3-e300-7585-3a43-9c9d3ed5b09e/data/default/testReplayEditsAfterRegionMovedWithMultiCF/25de72d74cf42cf025decee5a0d3cd6e 2024-11-24T08:29:54,505 DEBUG [RS_OPEN_REGION-regionserver/30c28c82771d:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1048): stopping wal replay for 25de72d74cf42cf025decee5a0d3cd6e 2024-11-24T08:29:54,505 DEBUG [RS_OPEN_REGION-regionserver/30c28c82771d:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1060): Cleaning up temporary data for 25de72d74cf42cf025decee5a0d3cd6e 2024-11-24T08:29:54,508 DEBUG [RS_OPEN_REGION-regionserver/30c28c82771d:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsAfterRegionMovedWithMultiCF descriptor;using region.getMemStoreFlushHeapSize/# of families (64.0 M)) instead. 2024-11-24T08:29:54,511 DEBUG [RS_OPEN_REGION-regionserver/30c28c82771d:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1093): writing seq id for 25de72d74cf42cf025decee5a0d3cd6e 2024-11-24T08:29:54,513 INFO [RS_OPEN_REGION-regionserver/30c28c82771d:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1114): Opened 25de72d74cf42cf025decee5a0d3cd6e; next sequenceid=18; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=69643872, jitterRate=0.03777456283569336}}}, FlushLargeStoresPolicy{flushSizeLowerBound=67108864} 2024-11-24T08:29:54,513 DEBUG [RS_OPEN_REGION-regionserver/30c28c82771d:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 25de72d74cf42cf025decee5a0d3cd6e 2024-11-24T08:29:54,514 DEBUG [RS_OPEN_REGION-regionserver/30c28c82771d:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1006): Region open journal for 25de72d74cf42cf025decee5a0d3cd6e: Running coprocessor pre-open hook at 1732436994478Writing region info on filesystem at 1732436994478Initializing all the Stores at 1732436994479 (+1 ms)Instantiating store for column family {NAME => 'cf1', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732436994479Instantiating store for column family {NAME => 'cf2', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732436994479Cleaning up temporary data from old regions at 1732436994505 (+26 ms)Running coprocessor post-open hooks at 1732436994513 (+8 ms)Region opened successfully at 1732436994514 (+1 ms) 2024-11-24T08:29:54,516 INFO [RS_OPEN_REGION-regionserver/30c28c82771d:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegionServer(2236): Post open deploy tasks for testReplayEditsAfterRegionMovedWithMultiCF,,1732436992313.25de72d74cf42cf025decee5a0d3cd6e., pid=12, masterSystemTime=1732436994472 2024-11-24T08:29:54,522 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=10 updating hbase:meta row=25de72d74cf42cf025decee5a0d3cd6e, regionState=OPEN, openSeqNum=18, regionLocation=30c28c82771d,38023,1732436977716 2024-11-24T08:29:54,524 DEBUG [RS_OPEN_REGION-regionserver/30c28c82771d:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegionServer(2266): Finished post open deploy task for testReplayEditsAfterRegionMovedWithMultiCF,,1732436992313.25de72d74cf42cf025decee5a0d3cd6e. 2024-11-24T08:29:54,524 INFO [RS_OPEN_REGION-regionserver/30c28c82771d:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] handler.AssignRegionHandler(153): Opened testReplayEditsAfterRegionMovedWithMultiCF,,1732436992313.25de72d74cf42cf025decee5a0d3cd6e. 2024-11-24T08:29:54,526 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=12, ppid=10, state=RUNNABLE, hasLock=false; OpenRegionProcedure 25de72d74cf42cf025decee5a0d3cd6e, server=30c28c82771d,38023,1732436977716 because future has completed 2024-11-24T08:29:54,530 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=12, resume processing ppid=10 2024-11-24T08:29:54,530 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=12, ppid=10, state=SUCCESS, hasLock=false; OpenRegionProcedure 25de72d74cf42cf025decee5a0d3cd6e, server=30c28c82771d,38023,1732436977716 in 208 msec 2024-11-24T08:29:54,532 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=10, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=25de72d74cf42cf025decee5a0d3cd6e, REOPEN/MOVE in 571 msec 2024-11-24T08:29:54,563 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-24T08:29:54,566 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39554, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-24T08:29:54,568 ERROR [Time-limited test {}] regionserver.HRegionServer(2442): ***** ABORTING region server 30c28c82771d,38023,1732436977716: testing ***** 2024-11-24T08:29:54,569 ERROR [Time-limited test {}] regionserver.HRegionServer(2447): RegionServer abort: loaded coprocessors are: [org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint] 2024-11-24T08:29:54,571 DEBUG [Time-limited test {}] util.JSONBean(135): Listing beans for java.lang:type=Memory 2024-11-24T08:29:54,572 DEBUG [Time-limited test {}] util.JSONBean(135): Listing beans for Hadoop:service=HBase,name=RegionServer,sub=IPC 2024-11-24T08:29:54,576 DEBUG [Time-limited test {}] util.JSONBean(135): Listing beans for Hadoop:service=HBase,name=RegionServer,sub=Replication 2024-11-24T08:29:54,577 DEBUG [Time-limited test {}] util.JSONBean(135): Listing beans for Hadoop:service=HBase,name=RegionServer,sub=Server 2024-11-24T08:29:54,585 INFO [Time-limited test {}] regionserver.HRegionServer(2451): Dump of metrics as JSON on abort: { "beans": [ { "name": "java.lang:type=Memory", "modelerType": "sun.management.MemoryImpl", "ObjectPendingFinalizationCount": 0, "HeapMemoryUsage": { "committed": 1048576000, "init": 1048576000, "max": 2306867200, "used": 337909760 }, "NonHeapMemoryUsage": { "committed": 171376640, "init": 7667712, "max": -1, "used": 168833272 }, "Verbose": false, "ObjectName": "java.lang:type=Memory" } ], "beans": [ { "name": "Hadoop:service=HBase,name=RegionServer,sub=IPC", "modelerType": "RegionServer,sub=IPC", "tag.Context": "regionserver", "tag.Hostname": "30c28c82771d", "queueSize": 0, "numCallsInGeneralQueue": 0, "numCallsInReplicationQueue": 0, "numCallsInBulkLoadQueue": 0, "numCallsInPriorityQueue": 0, "numCallsInMetaPriorityQueue": 0, "numOpenConnections": 0, "numActiveHandler": 0, "numActiveGeneralHandler": 0, "numActivePriorityHandler": 0, "numActiveReplicationHandler": 0, "numGeneralCallsDropped": 0, "numLifoModeSwitches": 0, "numCallsInWriteQueue": 0, "numActiveBulkLoadHandler": 0, "numCallsInReadQueue": 0, "numCallsInScanQueue": 0, "numActiveWriteHandler": 0, "numActiveReadHandler": 0, "numActiveScanHandler": 0, "nettyDirectMemoryUsage": 67108864, "nettyTotalPendingOutboundBytes": 0, "nettyMaxPendingOutboundBytes": 0, "receivedBytes": 2071, "exceptions.RegionMovedException": 0, "authenticationSuccesses": 0, "authorizationFailures": 0, "exceptions.requestTooBig": 0, "UnwritableTime_num_ops": 0, "UnwritableTime_min": 0, "UnwritableTime_max": 0, "UnwritableTime_mean": 0, "UnwritableTime_25th_percentile": 0, "UnwritableTime_median": 0, "UnwritableTime_75th_percentile": 0, "UnwritableTime_90th_percentile": 0, "UnwritableTime_95th_percentile": 0, "UnwritableTime_98th_percentile": 0, "UnwritableTime_99th_percentile": 0, "UnwritableTime_99.9th_percentile": 0, "exceptions.OutOfOrderScannerNextException": 0, "exceptions.rpcThrottling": 0, "exceptions.otherExceptions": 0, "ProcessCallTime_num_ops": 8, "ProcessCallTime_min": 1, "ProcessCallTime_max": 12, "ProcessCallTime_mean": 5, "ProcessCallTime_25th_percentile": 3, "ProcessCallTime_median": 6, "ProcessCallTime_75th_percentile": 9, "ProcessCallTime_90th_percentile": 10, "ProcessCallTime_95th_percentile": 11, "ProcessCallTime_98th_percentile": 11, "ProcessCallTime_99th_percentile": 11, "ProcessCallTime_99.9th_percentile": 11, "ProcessCallTime_TimeRangeCount_0-1": 8, "exceptions.callQueueTooBig": 0, "QueueCallTime_num_ops": 8, "QueueCallTime_min": 0, "QueueCallTime_max": 0, "QueueCallTime_mean": 0, "QueueCallTime_25th_percentile": 0, "QueueCallTime_median": 0, "QueueCallTime_75th_percentile": 0, "QueueCallTime_90th_percentile": 0, "QueueCallTime_95th_percentile": 0, "QueueCallTime_98th_percentile": 0, "QueueCallTime_99th_percentile": 0, "QueueCallTime_99.9th_percentile": 0, "QueueCallTime_TimeRangeCount_0-1": 8, "authenticationFailures": 0, "exceptions.multiResponseTooLarge": 0, "exceptions.callDropped": 0, "TotalCallTime_num_ops": 8, "TotalCallTime_min": 1, "TotalCallTime_max": 12, "TotalCallTime_mean": 5, "TotalCallTime_25th_percentile": 3, "TotalCallTime_median": 6, "TotalCallTime_75th_percentile": 9, "TotalCallTime_90th_percentile": 10, "TotalCallTime_95th_percentile": 11, "TotalCallTime_98th_percentile": 11, "TotalCallTime_99th_percentile": 11, "TotalCallTime_99.9th_percentile": 11, "TotalCallTime_TimeRangeCount_0-1": 8, "exceptions.RegionTooBusyException": 0, "exceptions.FailedSanityCheckException": 0, "ResponseSize_num_ops": 8, "ResponseSize_min": 0, "ResponseSize_max": 174, "ResponseSize_mean": 50, "ResponseSize_25th_percentile": 43, "ResponseSize_median": 87, "ResponseSize_75th_percentile": 130, "ResponseSize_90th_percentile": 156, "ResponseSize_95th_percentile": 165, "ResponseSize_98th_percentile": 170, "ResponseSize_99th_percentile": 172, "ResponseSize_99.9th_percentile": 173, "ResponseSize_SizeRangeCount_0-10": 8, "exceptions.UnknownScannerException": 0, "exceptions": 0, "maxOutboundBytesExceeded": 0, "authenticationFallbacks": 0, "exceptions.quotaExceeded": 0, "exceptions.callTimedOut": 0, "exceptions.NotServingRegionException": 0, "authorizationSuccesses": 0, "exceptions.ScannerResetException": 0, "RequestSize_num_ops": 8, "RequestSize_min": 89, "RequestSize_max": 121, "RequestSize_mean": 103, "RequestSize_25th_percentile": 97, "RequestSize_median": 105, "RequestSize_75th_percentile": 113, "RequestSize_90th_percentile": 117, "RequestSize_95th_percentile": 119, "RequestSize_98th_percentile": 120, "RequestSize_99th_percentile": 120, "RequestSize_99.9th_percentile": 120, "RequestSize_SizeRangeCount_0-10": 8, "sentBytes": 348 } ], "beans": [ { "name": "Hadoop:service=HBase,name=RegionServer,sub=Replication", "modelerType": "RegionServer,sub=Replication", "tag.Context": "regionserver", "tag.Hostname": "30c28c82771d", "source.shippedHFiles": 0, "Source.ageOfLastShippedOp_num_ops": 0, "Source.ageOfLastShippedOp_min": 0, "Source.ageOfLastShippedOp_max": 0, "Source.ageOfLastShippedOp_mean": 0, "Source.ageOfLastShippedOp_25th_percentile": 0, "Source.ageOfLastShippedOp_median": 0, "Source.ageOfLastShippedOp_75th_percentile": 0, "Source.ageOfLastShippedOp_90th_percentile": 0, "Source.ageOfLastShippedOp_95th_percentile": 0, "Source.ageOfLastShippedOp_98th_percentile": 0, "Source.ageOfLastShippedOp_99th_percentile": 0, "Source.ageOfLastShippedOp_99.9th_percentile": 0, "source.uncleanlyClosedLogs": 0, "source.closedLogsWithUnknownFileLength": 0, "source.walReaderEditsBufferUsage": 0, "source.repeatedLogFileBytes": 0, "source.sizeOfHFileRefsQueue": 0, "source.logReadInBytes": 0, "source.completedRecoverQueues": 0, "source.sizeOfLogQueue": 0, "source.restartedLogReading": 0, "source.failedRecoverQueues": 0, "source.ignoredUncleanlyClosedLogContentsInBytes": 0, "Sink.ageOfLastAppliedOp_num_ops": 0, "Sink.ageOfLastAppliedOp_min": 0, "Sink.ageOfLastAppliedOp_max": 0, "Sink.ageOfLastAppliedOp_mean": 0, "Sink.ageOfLastAppliedOp_25th_percentile": 0, "Sink.ageOfLastAppliedOp_median": 0, "Sink.ageOfLastAppliedOp_75th_percentile": 0, "Sink.ageOfLastAppliedOp_90th_percentile": 0, "Sink.ageOfLastAppliedOp_95th_percentile": 0, "Sink.ageOfLastAppliedOp_98th_percentile": 0, "Sink.ageOfLastAppliedOp_99th_percentile": 0, "Sink.ageOfLastAppliedOp_99.9th_percentile": 0, "source.logEditsRead": 0, "source.numInitializing": 0, "source.shippedOps": 0, "sink.appliedHFiles": 0, "source.logEditsFiltered": 0, "source.shippedBytes": 0, "sink.appliedOps": 0, "source.completedLogs": 0, "source.failedBatches": 0, "sink.failedBatches": 0, "source.shippedBatches": 0, "sink.appliedBatches": 0 } ], "beans": [ { "name": "Hadoop:service=HBase,name=RegionServer,sub=Server", "modelerType": "RegionServer,sub=Server", "tag.zookeeperQuorum": "127.0.0.1:59068", "tag.serverName": "30c28c82771d,40241,1732436977670", "tag.clusterId": "8962100e-ff2b-4a0c-9fd7-8de6412af7e9", "tag.Context": "regionserver", "tag.Hostname": "30c28c82771d", "regionCount": 1, "storeCount": 4, "hlogFileCount": 2, "hlogFileSize": 0, "storeFileCount": 0, "maxStoreFileCount": 0, "memStoreSize": 74, "memStoreHeapSize": 1248, "memStoreOffHeapSize": 0, "storeFileSize": 0, "storeFileSizeGrowthRate": 0.0, "maxStoreFileAge": 0, "minStoreFileAge": 0, "avgStoreFileAge": 0, "numReferenceFiles": 0, "regionServerStartTime": 1732436977670, "averageRegionSize": 74, "storeFileIndexSize": 0, "staticIndexSize": 0, "staticBloomSize": 0, "bloomFilterRequestsCount": 0, "bloomFilterNegativeResultsCount": 0, "bloomFilterEligibleRequestsCount": 0, "mutationsWithoutWALCount": 0, "mutationsWithoutWALSize": 0, "percentFilesLocal": 0.0, "percentFilesLocalSecondaryRegions": 0.0, "totalBytesRead": 10034, "localBytesRead": 10034, "shortCircuitBytesRead": 0, "zeroCopyBytesRead": 0, "splitQueueLength": 0, "compactionQueueLength": 0, "smallCompactionQueueLength": 0, "largeCompactionQueueLength": 0, "flushQueueLength": 0, "blockCacheFreeSize": 922070024, "blockCacheCount": 0, "blockCacheDataBlockCount": 0, "blockCacheSize": 676856, "blockCacheCountHitPercent": 0.0, "blockCacheExpressHitPercent": 0.0, "l1CacheSize": 676856, "l1CacheFreeSize": 922070024, "l1CacheCount": 0, "l1CacheEvictionCount": 0, "l1CacheHitCount": 0, "l1CacheMissCount": 0, "l1CacheHitRatio": 0.0, "l1CacheMissRatio": 0.0, "l2CacheSize": 0, "l2CacheFreeSize": 0, "l2CacheCount": 0, "l2CacheEvictionCount": 0, "l2CacheHitCount": 0, "l2CacheMissCount": 0, "l2CacheHitRatio": 0.0, "l2CacheMissRatio": 0.0, "mobFileCacheCount": 0, "mobFileCacheHitPercent": 0.0, "readRequestRatePerSecond": 1.4, "writeRequestRatePerSecond": 0.4, "ByteBuffAllocatorHeapAllocationBytes": 4782, "ByteBuffAllocatorPoolAllocationBytes": 199680, "ByteBuffAllocatorHeapAllocationRatio": 0.0, "ByteBuffAllocatorTotalBufferCount": 186, "ByteBuffAllocatorUsedBufferCount": 1, "activeScanners": 0, "totalRequestCount": 7, "totalRowActionRequestCount": 9, "readRequestCount": 7, "cpRequestCount": 0, "filteredReadRequestCount": 0, "writeRequestCount": 2, "rpcGetRequestCount": 1, "rpcFullScanRequestCount": 0, "rpcScanRequestCount": 4, "rpcMultiRequestCount": 0, "rpcMutateRequestCount": 2, "checkMutateFailedCount": 0, "checkMutatePassedCount": 0, "blockCacheHitCount": 0, "blockCacheHitCountPrimary": 0, "blockCacheHitCachingCount": 0, "blockCacheMissCount": 0, "blockCacheMissCountPrimary": 0, "blockCacheMissCachingCount": 0, "blockCacheEvictionCount": 0, "blockCacheEvictionCountPrimary": 0, "blockCacheFailedInsertionCount": 0, "blockCacheDataMissCount": 0, "blockCacheLeafIndexMissCount": 0, "blockCacheBloomChunkMissCount": 0, "blockCacheMetaMissCount": 0, "blockCacheRootIndexMissCount": 0, "blockCacheIntermediateIndexMissCount": 0, "blockCacheFileInfoMissCount": 0, "blockCacheGeneralBloomMetaMissCount": 0, "blockCacheDeleteFamilyBloomMissCount": 0, "blockCacheTrailerMissCount": 0, "blockCacheDataHitCount": 0, "blockCacheLeafIndexHitCount": 0, "blockCacheBloomChunkHitCount": 0, "blockCacheMetaHitCount": 0, "blockCacheRootIndexHitCount": 0, "blockCacheIntermediateIndexHitCount": 0, "blockCacheFileInfoHitCount": 0, "blockCacheGeneralBloomMetaHitCount": 0, "blockCacheDeleteFamilyBloomHitCount": 0, "blockCacheTrailerHitCount": 0, "updatesBlockedTime": 0, "flushedCellsCount": 0, "compactedCellsCount": 0, "majorCompactedCellsCount": 0, "flushedCellsSize": 0, "compactedCellsSize": 0, "majorCompactedCellsSize": 0, "cellsCountCompactedFromMob": 0, "cellsCountCompactedToMob": 0, "cellsSizeCompactedFromMob": 0, "cellsSizeCompactedToMob": 0, "mobFlushCount": 0, "mobFlushedCellsCount": 0, "mobFlushedCellsSize": 0, "mobScanCellsCount": 0, "mobScanCellsSize": 0, "mobFileCacheAccessCount": 0, "mobFileCacheMissCount": 0, "mobFileCacheEvictedCount": 0, "hedgedReads": 0, "hedgedReadWins": 0, "hedgedReadOpsInCurThread": 0, "blockedRequestCount": 0, "CheckAndMutate_num_ops": 0, "CheckAndMutate_min": 0, "CheckAndMutate_max": 0, "CheckAndMutate_mean": 0, "CheckAndMutate_25th_percentile": 0, "CheckAndMutate_median": 0, "CheckAndMutate_75th_percentile": 0, "CheckAndMutate_90th_percentile": 0, "CheckAndMutate_95th_percentile": 0, "CheckAndMutate_98th_percentile": 0, "CheckAndMutate_99th_percentile": 0, "CheckAndMutate_99.9th_percentile": 0, "MajorCompactionTime_num_ops": 0, "MajorCompactionTime_min": 0, "MajorCompactionTime_max": 0, "MajorCompactionTime_mean": 0, "MajorCompactionTime_25th_percentile": 0, "MajorCompactionTime_median": 0, "MajorCompactionTime_75th_percentile": 0, "MajorCompactionTime_90th_percentile": 0, "MajorCompactionTime_95th_percentile": 0, "MajorCompactionTime_98th_percentile": 0, "MajorCompactionTime_99th_percentile": 0, "MajorCompactionTime_99.9th_percentile": 0, "ScanTime_num_ops": 4, "ScanTime_min": 0, "ScanTime_max": 3, "ScanTime_mean": 0, "ScanTime_25th_percentile": 0, "ScanTime_median": 1, "ScanTime_75th_percentile": 2, "ScanTime_90th_percentile": 2, "ScanTime_95th_percentile": 2, "ScanTime_98th_percentile": 2, "ScanTime_99th_percentile": 2, "ScanTime_99.9th_percentile": 2, "ScanTime_TimeRangeCount_0-1": 4, "CheckAndMutateBlockBytesScanned_num_ops": 0, "CheckAndMutateBlockBytesScanned_min": 0, "CheckAndMutateBlockBytesScanned_max": 0, "CheckAndMutateBlockBytesScanned_mean": 0, "CheckAndMutateBlockBytesScanned_25th_percentile": 0, "CheckAndMutateBlockBytesScanned_median": 0, "CheckAndMutateBlockBytesScanned_75th_percentile": 0, "CheckAndMutateBlockBytesScanned_90th_percentile": 0, "CheckAndMutateBlockBytesScanned_95th_percentile": 0, "CheckAndMutateBlockBytesScanned_98th_percentile": 0, "CheckAndMutateBlockBytesScanned_99th_percentile": 0, "CheckAndMutateBlockBytesScanned_99.9th_percentile": 0, "Put_num_ops": 2, "Put_min": 2, "Put_max": 9, "Put_mean": 5, "Put_25th_percentile": 3, "Put_median": 5, "Put_75th_percentile": 7, "Put_90th_percentile": 8, "Put_95th_percentile": 8, "Put_98th_percentile": 8, "Put_99th_percentile": 8, "Put_99.9th_percentile": 8, "Put_TimeRangeCount_0-1": 2, "splitRequestCount": 0, "AppendBlockBytesScanned_num_ops": 0, "AppendBlockBytesScanned_min": 0, "AppendBlockBytesScanned_max": 0, "AppendBlockBytesScanned_mean": 0, "AppendBlockBytesScanned_25th_percentile": 0, "AppendBlockBytesScanned_median": 0, "AppendBlockBytesScanned_75th_percentile": 0, "AppendBlockBytesScanned_90th_percentile": 0, "AppendBlockBytesScanned_95th_percentile": 0, "AppendBlockBytesScanned_98th_percentile": 0, "AppendBlockBytesScanned_99th_percentile": 0, "AppendBlockBytesScanned_99.9th_percentile": 0, "PutBatch_num_ops": 0, "PutBatch_min": 0, "PutBatch_max": 0, "PutBatch_mean": 0, "PutBatch_25th_percentile": 0, "PutBatch_median": 0, "PutBatch_75th_percentile": 0, "PutBatch_90th_percentile": 0, "PutBatch_95th_percentile": 0, "PutBatch_98th_percentile": 0, "PutBatch_99th_percentile": 0, "PutBatch_99.9th_percentile": 0, "IncrementBlockBytesScanned_num_ops": 0, "IncrementBlockBytesScanned_min": 0, "IncrementBlockBytesScanned_max": 0, "IncrementBlockBytesScanned_mean": 0, "IncrementBlockBytesScanned_25th_percentile": 0, "IncrementBlockBytesScanned_median": 0, "IncrementBlockBytesScanned_75th_percentile": 0, "IncrementBlockBytesScanned_90th_percentile": 0, "IncrementBlockBytesScanned_95th_percentile": 0, "IncrementBlockBytesScanned_98th_percentile": 0, "IncrementBlockBytesScanned_99th_percentile": 0, "IncrementBlockBytesScanned_99.9th_percentile": 0, "SplitTime_num_ops": 0, "SplitTime_min": 0, "SplitTime_max": 0, "SplitTime_mean": 0, "SplitTime_25th_percentile": 0, "SplitTime_median": 0, "SplitTime_75th_percentile": 0, "SplitTime_90th_percentile": 0, "SplitTime_95th_percentile": 0, "SplitTime_98th_percentile": 0, "SplitTime_99th_percentile": 0, "SplitTime_99.9th_percentile": 0, "GetBlockBytesScanned_num_ops": 0, "GetBlockBytesScanned_min": 0, "GetBlockBytesScanned_max": 0, "GetBlockBytesScanned_mean": 0, "GetBlockBytesScanned_25th_percentile": 0, "GetBlockBytesScanned_median": 0, "GetBlockBytesScanned_75th_percentile": 0, "GetBlockBytesScanned_90th_percentile": 0, "GetBlockBytesScanned_95th_percentile": 0, "GetBlockBytesScanned_98th_percentile": 0, "GetBlockBytesScanned_99th_percentile": 0, "GetBlockBytesScanned_99.9th_percentile": 0, "majorCompactedInputBytes": 0, "slowAppendCount": 0, "flushedOutputBytes": 0, "Replay_num_ops": 0, "Replay_min": 0, "Replay_max": 0, "Replay_mean": 0, "Replay_25th_percentile": 0, "Replay_median": 0, "Replay_75th_percentile": 0, "Replay_90th_percentile": 0, "Replay_95th_percentile": 0, "Replay_98th_percentile": 0, "Replay_99th_percentile": 0, "Replay_99.9th_percentile": 0, "MajorCompactionInputSize_num_ops": 0, "MajorCompactionInputSize_min": 0, "MajorCompactionInputSize_max": 0, "MajorCompactionInputSize_mean": 0, "MajorCompactionInputSize_25th_percentile": 0, "MajorCompactionInputSize_median": 0, "MajorCompactionInputSize_75th_percentile": 0, "MajorCompactionInputSize_90th_percentile": 0, "MajorCompactionInputSize_95th_percentile": 0, "MajorCompactionInputSize_98th_percentile": 0, "MajorCompactionInputSize_99th_percentile": 0, "MajorCompactionInputSize_99.9th_percentile": 0, "pauseInfoThresholdExceeded": 0, "CheckAndDelete_num_ops": 0, "CheckAndDelete_min": 0, "CheckAndDelete_max": 0, "CheckAndDelete_mean": 0, "CheckAndDelete_25th_percentile": 0, "CheckAndDelete_median": 0, "CheckAndDelete_75th_percentile": 0, "CheckAndDelete_90th_percentile": 0, "CheckAndDelete_95th_percentile": 0, "CheckAndDelete_98th_percentile": 0, "CheckAndDelete_99th_percentile": 0, "CheckAndDelete_99.9th_percentile": 0, "CompactionInputSize_num_ops": 0, "CompactionInputSize_min": 0, "CompactionInputSize_max": 0, "CompactionInputSize_mean": 0, "CompactionInputSize_25th_percentile": 0, "CompactionInputSize_median": 0, "CompactionInputSize_75th_percentile": 0, "CompactionInputSize_90th_percentile": 0, "CompactionInputSize_95th_percentile": 0, "CompactionInputSize_98th_percentile": 0, "CompactionInputSize_99th_percentile": 0, "CompactionInputSize_99.9th_percentile": 0, "flushedMemstoreBytes": 0, "majorCompactedOutputBytes": 0, "slowPutCount": 0, "compactedInputBytes": 0, "FlushOutputSize_num_ops": 0, "FlushOutputSize_min": 0, "FlushOutputSize_max": 0, "FlushOutputSize_mean": 0, "FlushOutputSize_25th_percentile": 0, "FlushOutputSize_median": 0, "FlushOutputSize_75th_percentile": 0, "FlushOutputSize_90th_percentile": 0, "FlushOutputSize_95th_percentile": 0, "FlushOutputSize_98th_percentile": 0, "FlushOutputSize_99th_percentile": 0, "FlushOutputSize_99.9th_percentile": 0, "PauseTimeWithGc_num_ops": 0, "PauseTimeWithGc_min": 0, "PauseTimeWithGc_max": 0, "PauseTimeWithGc_mean": 0, "PauseTimeWithGc_25th_percentile": 0, "PauseTimeWithGc_median": 0, "PauseTimeWithGc_75th_percentile": 0, "PauseTimeWithGc_90th_percentile": 0, "PauseTimeWithGc_95th_percentile": 0, "PauseTimeWithGc_98th_percentile": 0, "PauseTimeWithGc_99th_percentile": 0, "PauseTimeWithGc_99.9th_percentile": 0, "compactedOutputBytes": 0, "pauseWarnThresholdExceeded": 0, "ScanBlockBytesScanned_num_ops": 0, "ScanBlockBytesScanned_min": 0, "ScanBlockBytesScanned_max": 0, "ScanBlockBytesScanned_mean": 0, "ScanBlockBytesScanned_25th_percentile": 0, "ScanBlockBytesScanned_median": 0, "ScanBlockBytesScanned_75th_percentile": 0, "ScanBlockBytesScanned_90th_percentile": 0, "ScanBlockBytesScanned_95th_percentile": 0, "ScanBlockBytesScanned_98th_percentile": 0, "ScanBlockBytesScanned_99th_percentile": 0, "ScanBlockBytesScanned_99.9th_percentile": 0, "Increment_num_ops": 0, "Increment_min": 0, "Increment_max": 0, "Increment_mean": 0, "Increment_25th_percentile": 0, "Increment_median": 0, "Increment_75th_percentile": 0, "Increment_90th_percentile": 0, "Increment_95th_percentile": 0, "Increment_98th_percentile": 0, "Increment_99th_percentile": 0, "Increment_99.9th_percentile": 0, "Delete_num_ops": 0, "Delete_min": 0, "Delete_max": 0, "Delete_mean": 0, "Delete_25th_percentile": 0, "Delete_median": 0, "Delete_75th_percentile": 0, "Delete_90th_percentile": 0, "Delete_95th_percentile": 0, "Delete_98th_percentile": 0, "Delete_99th_percentile": 0, "Delete_99.9th_percentile": 0, "DeleteBatch_num_ops": 0, "DeleteBatch_min": 0, "DeleteBatch_max": 0, "DeleteBatch_mean": 0, "DeleteBatch_25th_percentile": 0, "DeleteBatch_median": 0, "DeleteBatch_75th_percentile": 0, "DeleteBatch_90th_percentile": 0, "DeleteBatch_95th_percentile": 0, "DeleteBatch_98th_percentile": 0, "DeleteBatch_99th_percentile": 0, "DeleteBatch_99.9th_percentile": 0, "blockBytesScannedCount": 0, "FlushMemstoreSize_num_ops": 0, "FlushMemstoreSize_min": 0, "FlushMemstoreSize_max": 0, "FlushMemstoreSize_mean": 0, "FlushMemstoreSize_25th_percentile": 0, "FlushMemstoreSize_median": 0, "FlushMemstoreSize_75th_percentile": 0, "FlushMemstoreSize_90th_percentile": 0, "FlushMemstoreSize_95th_percentile": 0, "FlushMemstoreSize_98th_percentile": 0, "FlushMemstoreSize_99th_percentile": 0, "FlushMemstoreSize_99.9th_percentile": 0, "CompactionInputFileCount_num_ops": 0, "CompactionInputFileCount_min": 0, "CompactionInputFileCount_max": 0, "CompactionInputFileCount_mean": 0, "CompactionInputFileCount_25th_percentile": 0, "CompactionInputFileCount_median": 0, "CompactionInputFileCount_75th_percentile": 0, "CompactionInputFileCount_90th_percentile": 0, "CompactionInputFileCount_95th_percentile": 0, "CompactionInputFileCount_98th_percentile": 0, "CompactionInputFileCount_99th_percentile": 0, "CompactionInputFileCount_99.9th_percentile": 0, "CompactionTime_num_ops": 0, "CompactionTime_min": 0, "CompactionTime_max": 0, "CompactionTime_mean": 0, "CompactionTime_25th_percentile": 0, "CompactionTime_median": 0, "CompactionTime_75th_percentile": 0, "CompactionTime_90th_percentile": 0, "CompactionTime_95th_percentile": 0, "CompactionTime_98th_percentile": 0, "CompactionTime_99th_percentile": 0, "CompactionTime_99.9th_percentile": 0, "Get_num_ops": 1, "Get_min": 2, "Get_max": 2, "Get_mean": 2, "Get_25th_percentile": 2, "Get_median": 2, "Get_75th_percentile": 2, "Get_90th_percentile": 2, "Get_95th_percentile": 2, "Get_98th_percentile": 2, "Get_99th_percentile": 2, "Get_99.9th_percentile": 2, "Get_TimeRangeCount_0-1": 1, "MajorCompactionInputFileCount_num_ops": 0, "MajorCompactionInputFileCount_min": 0, "MajorCompactionInputFileCount_max": 0, "MajorCompactionInputFileCount_mean": 0, "MajorCompactionInputFileCount_25th_percentile": 0, "MajorCompactionInputFileCount_median": 0, "MajorCompactionInputFileCount_75th_percentile": 0, "MajorCompactionInputFileCount_90th_percentile": 0, "MajorCompactionInputFileCount_95th_percentile": 0, "MajorCompactionInputFileCount_98th_percentile": 0, "MajorCompactionInputFileCount_99th_percentile": 0, "MajorCompactionInputFileCount_99.9th_percentile": 0, "scannerLeaseExpiredCount": 0, "CheckAndPut_num_ops": 0, "CheckAndPut_min": 0, "CheckAndPut_max": 0, "CheckAndPut_mean": 0, "CheckAndPut_25th_percentile": 0, "CheckAndPut_median": 0, "CheckAndPut_75th_percentile": 0, "CheckAndPut_90th_percentile": 0, "CheckAndPut_95th_percentile": 0, "CheckAndPut_98th_percentile": 0, "CheckAndPut_99th_percentile": 0, "CheckAndPut_99.9th_percentile": 0, "MajorCompactionOutputSize_num_ops": 0, "MajorCompactionOutputSize_min": 0, "MajorCompactionOutputSize_max": 0, "MajorCompactionOutputSize_mean": 0, "MajorCompactionOutputSize_25th_percentile": 0, "MajorCompactionOutputSize_median": 0, "MajorCompactionOutputSize_75th_percentile": 0, "MajorCompactionOutputSize_90th_percentile": 0, "MajorCompactionOutputSize_95th_percentile": 0, "MajorCompactionOutputSize_98th_percentile": 0, "MajorCompactionOutputSize_99th_percentile": 0, "MajorCompactionOutputSize_99.9th_percentile": 0, "CompactionOutputFileCount_num_ops": 0, "CompactionOutputFileCount_min": 0, "CompactionOutputFileCount_max": 0, "CompactionOutputFileCount_mean": 0, "CompactionOutputFileCount_25th_percentile": 0, "CompactionOutputFileCount_median": 0, "CompactionOutputFileCount_75th_percentile": 0, "CompactionOutputFileCount_90th_percentile": 0, "CompactionOutputFileCount_95th_percentile": 0, "CompactionOutputFileCount_98th_percentile": 0, "CompactionOutputFileCount_99th_percentile": 0, "CompactionOutputFileCount_99.9th_percentile": 0, "slowDeleteCount": 0, "FlushTime_num_ops": 0, "FlushTime_min": 0, "FlushTime_max": 0, "FlushTime_mean": 0, "FlushTime_25th_percentile": 0, "FlushTime_median": 0, "FlushTime_75th_percentile": 0, "FlushTime_90th_percentile": 0, "FlushTime_95th_percentile": 0, "FlushTime_98th_percentile": 0, "FlushTime_99th_percentile": 0, "FlushTime_99.9th_percentile": 0, "splitSuccessCount": 0, "MajorCompactionOutputFileCount_num_ops": 0, "MajorCompactionOutputFileCount_min": 0, "MajorCompactionOutputFileCount_max": 0, "MajorCompactionOutputFileCount_mean": 0, "MajorCompactionOutputFileCount_25th_percentile": 0, "MajorCompactionOutputFileCount_median": 0, "MajorCompactionOutputFileCount_75th_percentile": 0, "MajorCompactionOutputFileCount_90th_percentile": 0, "MajorCompactionOutputFileCount_95th_percentile": 0, "MajorCompactionOutputFileCount_98th_percentile": 0, "MajorCompactionOutputFileCount_99th_percentile": 0, "MajorCompactionOutputFileCount_99.9th_percentile": 0, "slowGetCount": 0, "ScanSize_num_ops": 4, "ScanSize_min": 0, "ScanSize_max": 144, "ScanSize_mean": 72, "ScanSize_25th_percentile": 36, "ScanSize_median": 72, "ScanSize_75th_percentile": 108, "ScanSize_90th_percentile": 129, "ScanSize_95th_percentile": 136, "ScanSize_98th_percentile": 141, "ScanSize_99th_percentile": 142, "ScanSize_99.9th_percentile": 143, "ScanSize_SizeRangeCount_0-10": 4, "CompactionOutputSize_num_ops": 0, "CompactionOutputSize_min": 0, "CompactionOutputSize_max": 0, "CompactionOutputSize_mean": 0, "CompactionOutputSize_25th_percentile": 0, "CompactionOutputSize_median": 0, "CompactionOutputSize_75th_percentile": 0, "CompactionOutputSize_90th_percentile": 0, "CompactionOutputSize_95th_percentile": 0, "CompactionOutputSize_98th_percentile": 0, "CompactionOutputSize_99th_percentile": 0, "CompactionOutputSize_99.9th_percentile": 0, "PauseTimeWithoutGc_num_ops": 0, "PauseTimeWithoutGc_min": 0, "PauseTimeWithoutGc_max": 0, "PauseTimeWithoutGc_mean": 0, "PauseTimeWithoutGc_25th_percentile": 0, "PauseTimeWithoutGc_median": 0, "PauseTimeWithoutGc_75th_percentile": 0, "PauseTimeWithoutGc_90th_percentile": 0, "PauseTimeWithoutGc_95th_percentile": 0, "PauseTimeWithoutGc_98th_percentile": 0, "PauseTimeWithoutGc_99th_percentile": 0, "PauseTimeWithoutGc_99.9th_percentile": 0, "slowIncrementCount": 0, "Append_num_ops": 0, "Append_min": 0, "Append_max": 0, "Append_mean": 0, "Append_25th_percentile": 0, "Append_median": 0, "Append_75th_percentile": 0, "Append_90th_percentile": 0, "Append_95th_percentile": 0, "Append_98th_percentile": 0, "Append_99th_percentile": 0, "Append_99.9th_percentile": 0, "Bulkload_count": 0, "Bulkload_mean_rate": 0.0, "Bulkload_1min_rate": 0.0, "Bulkload_5min_rate": 0.0, "Bulkload_15min_rate": 0.0, "Bulkload_num_ops": 0, "Bulkload_min": 0, "Bulkload_max": 0, "Bulkload_mean": 0, "Bulkload_25th_percentile": 0, "Bulkload_median": 0, "Bulkload_75th_percentile": 0, "Bulkload_90th_percentile": 0, "Bulkload_95th_percentile": 0, "Bulkload_98th_percentile": 0, "Bulkload_99th_percentile": 0, "Bulkload_99.9th_percentile": 0 } ] } 2024-11-24T08:29:54,588 WARN [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37719 {}] master.MasterRpcServices(700): 30c28c82771d,38023,1732436977716 reported a fatal error: ***** ABORTING region server 30c28c82771d,38023,1732436977716: testing ***** 2024-11-24T08:29:54,590 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '30c28c82771d,38023,1732436977716' ***** 2024-11-24T08:29:54,591 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: testing 2024-11-24T08:29:54,591 INFO [RS:2;30c28c82771d:38023 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-24T08:29:54,591 INFO [RS:2;30c28c82771d:38023 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager abruptly. 2024-11-24T08:29:54,591 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-24T08:29:54,591 INFO [RS:2;30c28c82771d:38023 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager abruptly. 2024-11-24T08:29:54,592 INFO [RS:2;30c28c82771d:38023 {}] regionserver.HRegionServer(3091): Received CLOSE for 25de72d74cf42cf025decee5a0d3cd6e 2024-11-24T08:29:54,592 INFO [RS:2;30c28c82771d:38023 {}] regionserver.HRegionServer(956): aborting server 30c28c82771d,38023,1732436977716 2024-11-24T08:29:54,592 INFO [RS:2;30c28c82771d:38023 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-24T08:29:54,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40117 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Get size: 140 connection: 172.17.0.2:46008 deadline: 1732437054592, exception=org.apache.hadoop.hbase.exceptions.RegionMovedException: Region moved to: hostname=30c28c82771d port=38023 startCode=1732436977716. As of locationSeqNum=12. 2024-11-24T08:29:54,592 INFO [RS:2;30c28c82771d:38023 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:2;30c28c82771d:38023. 2024-11-24T08:29:54,593 DEBUG [RS:2;30c28c82771d:38023 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-24T08:29:54,593 DEBUG [RS_CLOSE_REGION-regionserver/30c28c82771d:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 25de72d74cf42cf025decee5a0d3cd6e, disabling compactions & flushes 2024-11-24T08:29:54,593 INFO [RS_CLOSE_REGION-regionserver/30c28c82771d:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testReplayEditsAfterRegionMovedWithMultiCF,,1732436992313.25de72d74cf42cf025decee5a0d3cd6e. 2024-11-24T08:29:54,593 DEBUG [RPCClient-NioEventLoopGroup-6-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1732436992313.25de72d74cf42cf025decee5a0d3cd6e., hostname=30c28c82771d,40117,1732436977557, seqNum=5 , the old value is region=testReplayEditsAfterRegionMovedWithMultiCF,,1732436992313.25de72d74cf42cf025decee5a0d3cd6e., hostname=30c28c82771d,40117,1732436977557, seqNum=5, error=org.apache.hadoop.hbase.exceptions.RegionMovedException: Region moved to: hostname=30c28c82771d port=38023 startCode=1732436977716. As of locationSeqNum=12. 2024-11-24T08:29:54,593 DEBUG [RS_CLOSE_REGION-regionserver/30c28c82771d:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1732436992313.25de72d74cf42cf025decee5a0d3cd6e. 2024-11-24T08:29:54,593 DEBUG [RPCClient-NioEventLoopGroup-6-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1732436992313.25de72d74cf42cf025decee5a0d3cd6e., hostname=30c28c82771d,40117,1732436977557, seqNum=5 is org.apache.hadoop.hbase.exceptions.RegionMovedException: Region moved to: hostname=30c28c82771d port=38023 startCode=1732436977716. As of locationSeqNum=12. 2024-11-24T08:29:54,593 DEBUG [RS:2;30c28c82771d:38023 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T08:29:54,593 DEBUG [RS_CLOSE_REGION-regionserver/30c28c82771d:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1732436992313.25de72d74cf42cf025decee5a0d3cd6e. after waiting 0 ms 2024-11-24T08:29:54,593 DEBUG [RPCClient-NioEventLoopGroup-6-7 {}] client.AsyncRegionLocatorHelper(84): Try updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1732436992313.25de72d74cf42cf025decee5a0d3cd6e., hostname=30c28c82771d,40117,1732436977557, seqNum=5 with the new location region=testReplayEditsAfterRegionMovedWithMultiCF,,1732436992313.25de72d74cf42cf025decee5a0d3cd6e., hostname=30c28c82771d,38023,1732436977716, seqNum=12 constructed by org.apache.hadoop.hbase.exceptions.RegionMovedException: Region moved to: hostname=30c28c82771d port=38023 startCode=1732436977716. As of locationSeqNum=12. 2024-11-24T08:29:54,593 DEBUG [RS_CLOSE_REGION-regionserver/30c28c82771d:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsAfterRegionMovedWithMultiCF,,1732436992313.25de72d74cf42cf025decee5a0d3cd6e. 2024-11-24T08:29:54,593 INFO [RS:2;30c28c82771d:38023 {}] regionserver.HRegionServer(1321): Waiting on 1 regions to close 2024-11-24T08:29:54,594 DEBUG [RS:2;30c28c82771d:38023 {}] regionserver.HRegionServer(1325): Online Regions={25de72d74cf42cf025decee5a0d3cd6e=testReplayEditsAfterRegionMovedWithMultiCF,,1732436992313.25de72d74cf42cf025decee5a0d3cd6e.} 2024-11-24T08:29:54,594 DEBUG [RS:2;30c28c82771d:38023 {}] regionserver.HRegionServer(1351): Waiting on 25de72d74cf42cf025decee5a0d3cd6e 2024-11-24T08:29:54,606 INFO [regionserver/30c28c82771d:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-24T08:29:54,623 INFO [RS_CLOSE_REGION-regionserver/30c28c82771d:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testReplayEditsAfterRegionMovedWithMultiCF,,1732436992313.25de72d74cf42cf025decee5a0d3cd6e. 2024-11-24T08:29:54,623 DEBUG [RS_CLOSE_REGION-regionserver/30c28c82771d:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 25de72d74cf42cf025decee5a0d3cd6e: Waiting for close lock at 1732436994593Running coprocessor pre-close hooks at 1732436994593Disabling compacts and flushes for region at 1732436994593Disabling writes for close at 1732436994593Writing region close event to WAL at 1732436994623 (+30 ms)Running coprocessor post-close hooks at 1732436994623Closed at 1732436994623 2024-11-24T08:29:54,624 DEBUG [RS_CLOSE_REGION-regionserver/30c28c82771d:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testReplayEditsAfterRegionMovedWithMultiCF,,1732436992313.25de72d74cf42cf025decee5a0d3cd6e. 2024-11-24T08:29:54,705 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.regionserver.RegionServerAbortedException: Server 30c28c82771d,38023,1732436977716 aborting at org.apache.hadoop.hbase.ipc.ServerRpcConnection.processRequest(ServerRpcConnection.java:564) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.ServerRpcConnection.processOneRpc(ServerRpcConnection.java:364) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyServerRpcConnection.process(NettyServerRpcConnection.java:89) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcServerRequestDecoder.channelRead0(NettyRpcServerRequestDecoder.java:56) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcServerRequestDecoder.channelRead0(NettyRpcServerRequestDecoder.java:31) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollStreamChannel$EpollStreamUnsafe.epollInReady(AbstractEpollStreamChannel.java:799) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:501) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T08:29:54,707 DEBUG [RPCClient-NioEventLoopGroup-6-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1732436992313.25de72d74cf42cf025decee5a0d3cd6e., hostname=30c28c82771d,38023,1732436977716, seqNum=12 , the old value is region=testReplayEditsAfterRegionMovedWithMultiCF,,1732436992313.25de72d74cf42cf025decee5a0d3cd6e., hostname=30c28c82771d,38023,1732436977716, seqNum=12, error=org.apache.hadoop.hbase.regionserver.RegionServerAbortedException: Server 30c28c82771d,38023,1732436977716 aborting 2024-11-24T08:29:54,707 DEBUG [RPCClient-NioEventLoopGroup-6-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1732436992313.25de72d74cf42cf025decee5a0d3cd6e., hostname=30c28c82771d,38023,1732436977716, seqNum=12 is org.apache.hadoop.hbase.regionserver.RegionServerAbortedException: Server 30c28c82771d,38023,1732436977716 aborting 2024-11-24T08:29:54,707 DEBUG [RPCClient-NioEventLoopGroup-6-5 {}] client.AsyncRegionLocatorHelper(88): Try removing region=testReplayEditsAfterRegionMovedWithMultiCF,,1732436992313.25de72d74cf42cf025decee5a0d3cd6e., hostname=30c28c82771d,38023,1732436977716, seqNum=12 from cache 2024-11-24T08:29:54,794 INFO [RS:2;30c28c82771d:38023 {}] regionserver.HRegionServer(976): stopping server 30c28c82771d,38023,1732436977716; all regions closed. 2024-11-24T08:29:54,798 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40533 is added to blk_1073741833_1009 (size=1677) 2024-11-24T08:29:54,798 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34459 is added to blk_1073741833_1009 (size=1677) 2024-11-24T08:29:54,798 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33555 is added to blk_1073741833_1009 (size=1677) 2024-11-24T08:29:54,799 DEBUG [RS:2;30c28c82771d:38023 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T08:29:54,799 INFO [RS:2;30c28c82771d:38023 {}] regionserver.LeaseManager(133): Closed leases 2024-11-24T08:29:54,799 INFO [RS:2;30c28c82771d:38023 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-24T08:29:54,799 INFO [RS:2;30c28c82771d:38023 {}] hbase.ChoreService(370): Chore service for: regionserver/30c28c82771d:0 had [ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-11-24T08:29:54,799 INFO [regionserver/30c28c82771d:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-24T08:29:54,799 INFO [RS:2;30c28c82771d:38023 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-24T08:29:54,799 INFO [RS:2;30c28c82771d:38023 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-24T08:29:54,799 INFO [RS:2;30c28c82771d:38023 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-24T08:29:54,800 INFO [RS:2;30c28c82771d:38023 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-24T08:29:54,800 INFO [RS:2;30c28c82771d:38023 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:38023 2024-11-24T08:29:54,807 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38023-0x10149184c260003, quorum=127.0.0.1:59068, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/30c28c82771d,38023,1732436977716 2024-11-24T08:29:54,807 INFO [RS:2;30c28c82771d:38023 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-24T08:29:54,807 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37719-0x10149184c260000, quorum=127.0.0.1:59068, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-24T08:29:54,809 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [30c28c82771d,38023,1732436977716] 2024-11-24T08:29:54,810 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/30c28c82771d,38023,1732436977716 already deleted, retry=false 2024-11-24T08:29:54,811 INFO [RegionServerTracker-0 {}] master.ServerManager(695): Processing expiration of 30c28c82771d,38023,1732436977716 on 30c28c82771d,37719,1732436976835 2024-11-24T08:29:54,816 DEBUG [RegionServerTracker-0 {}] procedure2.ProcedureExecutor(1139): Stored pid=13, state=RUNNABLE:SERVER_CRASH_START, hasLock=false; ServerCrashProcedure 30c28c82771d,38023,1732436977716, splitWal=true, meta=false 2024-11-24T08:29:54,819 INFO [PEWorker-1 {}] procedure.ServerCrashProcedure(169): Start pid=13, state=RUNNABLE:SERVER_CRASH_START, hasLock=true; ServerCrashProcedure 30c28c82771d,38023,1732436977716, splitWal=true, meta=false 2024-11-24T08:29:54,819 INFO [RegionServerTracker-0 {}] assignment.AssignmentManager(1999): Scheduled ServerCrashProcedure pid=13 for 30c28c82771d,38023,1732436977716 (carryingMeta=false) 30c28c82771d,38023,1732436977716/CRASHED/regionCount=1/lock=java.util.concurrent.locks.ReentrantReadWriteLock@57267bbb[Write locks = 1, Read locks = 0], oldState=ONLINE. 2024-11-24T08:29:54,821 INFO [PEWorker-1 {}] procedure.ServerCrashProcedure(207): 30c28c82771d,38023,1732436977716 had 1 regions 2024-11-24T08:29:54,822 INFO [PEWorker-1 {}] procedure.ServerCrashProcedure(339): Splitting WALs pid=13, state=RUNNABLE:SERVER_CRASH_SPLIT_LOGS, hasLock=true; ServerCrashProcedure 30c28c82771d,38023,1732436977716, splitWal=true, meta=false, isMeta: false 2024-11-24T08:29:54,824 DEBUG [PEWorker-1 {}] master.MasterWalManager(329): Renamed region directory: hdfs://localhost:46165/user/jenkins/test-data/7507fec3-e300-7585-3a43-9c9d3ed5b09e/WALs/30c28c82771d,38023,1732436977716-splitting 2024-11-24T08:29:54,825 INFO [PEWorker-1 {}] master.SplitWALManager(105): 30c28c82771d,38023,1732436977716 WAL count=1, meta=false 2024-11-24T08:29:54,827 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=14, ppid=13, state=RUNNABLE:ACQUIRE_SPLIT_WAL_WORKER, hasLock=false; SplitWALProcedure 30c28c82771d%2C38023%2C1732436977716.1732436979045}] 2024-11-24T08:29:54,833 DEBUG [PEWorker-2 {}] master.SplitWALManager(158): Acquired split WAL worker=30c28c82771d,40241,1732436977670 2024-11-24T08:29:54,835 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=15, ppid=14, state=RUNNABLE, hasLock=false; SplitWALRemoteProcedure 30c28c82771d%2C38023%2C1732436977716.1732436979045, worker=30c28c82771d,40241,1732436977670}] 2024-11-24T08:29:54,909 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38023-0x10149184c260003, quorum=127.0.0.1:59068, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-24T08:29:54,909 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38023-0x10149184c260003, quorum=127.0.0.1:59068, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-24T08:29:54,910 INFO [RS:2;30c28c82771d:38023 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-24T08:29:54,910 INFO [RS:2;30c28c82771d:38023 {}] regionserver.HRegionServer(1031): Exiting; stopping=30c28c82771d,38023,1732436977716; zookeeper connection closed. 2024-11-24T08:29:54,910 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@55a8332a {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@55a8332a 2024-11-24T08:29:54,920 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testReplayEditsAfterRegionMovedWithMultiCF', row='r1', locateType=CURRENT is [region=testReplayEditsAfterRegionMovedWithMultiCF,,1732436992313.25de72d74cf42cf025decee5a0d3cd6e., hostname=30c28c82771d,38023,1732436977716, seqNum=18] 2024-11-24T08:29:54,922 WARN [RPCClient-NioEventLoopGroup-6-5 {}] ipc.NettyRpcConnection$2(409): Exception encountered while connecting to the server 30c28c82771d:38023 org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: Connection refused: 30c28c82771d/172.17.0.2:38023 Caused by: java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hbase.thirdparty.io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(NioSocketChannel.java:336) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:339) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T08:29:54,923 DEBUG [RPCClient-NioEventLoopGroup-6-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1732436992313.25de72d74cf42cf025decee5a0d3cd6e., hostname=30c28c82771d,38023,1732436977716, seqNum=18 , the old value is region=testReplayEditsAfterRegionMovedWithMultiCF,,1732436992313.25de72d74cf42cf025decee5a0d3cd6e., hostname=30c28c82771d,38023,1732436977716, seqNum=18, error=java.net.ConnectException: Call to address=30c28c82771d:38023 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: Connection refused: 30c28c82771d/172.17.0.2:38023 2024-11-24T08:29:54,923 DEBUG [RPCClient-NioEventLoopGroup-6-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1732436992313.25de72d74cf42cf025decee5a0d3cd6e., hostname=30c28c82771d,38023,1732436977716, seqNum=18 is java.net.ConnectException: Connection refused 2024-11-24T08:29:54,923 DEBUG [RPCClient-NioEventLoopGroup-6-5 {}] client.AsyncRegionLocatorHelper(88): Try removing region=testReplayEditsAfterRegionMovedWithMultiCF,,1732436992313.25de72d74cf42cf025decee5a0d3cd6e., hostname=30c28c82771d,38023,1732436977716, seqNum=18 from cache 2024-11-24T08:29:54,923 DEBUG [RPCClient-NioEventLoopGroup-6-5 {}] ipc.FailedServers(52): Added failed server with address 30c28c82771d:38023 to list caused by org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: Connection refused: 30c28c82771d/172.17.0.2:38023 2024-11-24T08:29:54,997 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40241 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SplitWALCallable, pid=15 2024-11-24T08:29:55,015 INFO [RS_LOG_REPLAY_OPS-regionserver/30c28c82771d:0-0 {event_type=RS_LOG_REPLAY, pid=15}] wal.WALSplitter(299): Splitting hdfs://localhost:46165/user/jenkins/test-data/7507fec3-e300-7585-3a43-9c9d3ed5b09e/WALs/30c28c82771d,38023,1732436977716-splitting/30c28c82771d%2C38023%2C1732436977716.1732436979045, size=1.6 K (1677bytes) 2024-11-24T08:29:55,015 INFO [RS_LOG_REPLAY_OPS-regionserver/30c28c82771d:0-0 {event_type=RS_LOG_REPLAY, pid=15}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:46165/user/jenkins/test-data/7507fec3-e300-7585-3a43-9c9d3ed5b09e/WALs/30c28c82771d,38023,1732436977716-splitting/30c28c82771d%2C38023%2C1732436977716.1732436979045 2024-11-24T08:29:55,016 INFO [RS_LOG_REPLAY_OPS-regionserver/30c28c82771d:0-0 {event_type=RS_LOG_REPLAY, pid=15}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:46165/user/jenkins/test-data/7507fec3-e300-7585-3a43-9c9d3ed5b09e/WALs/30c28c82771d,38023,1732436977716-splitting/30c28c82771d%2C38023%2C1732436977716.1732436979045 after 1ms 2024-11-24T08:29:55,018 DEBUG [RS_LOG_REPLAY_OPS-regionserver/30c28c82771d:0-0 {event_type=RS_LOG_REPLAY, pid=15}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:46165/user/jenkins/test-data/7507fec3-e300-7585-3a43-9c9d3ed5b09e/WALs/30c28c82771d,38023,1732436977716-splitting/30c28c82771d%2C38023%2C1732436977716.1732436979045: isRecoveredEdits=false, hasTagCompression=true, hasValueCompression=false, valueCompressionType=null 2024-11-24T08:29:55,019 INFO [RS_LOG_REPLAY_OPS-regionserver/30c28c82771d:0-0 {event_type=RS_LOG_REPLAY, pid=15}] wal.WALSplitter(310): Open hdfs://localhost:46165/user/jenkins/test-data/7507fec3-e300-7585-3a43-9c9d3ed5b09e/WALs/30c28c82771d,38023,1732436977716-splitting/30c28c82771d%2C38023%2C1732436977716.1732436979045 took 4ms 2024-11-24T08:29:55,028 DEBUG [RS_LOG_REPLAY_OPS-regionserver/30c28c82771d:0-0 {event_type=RS_LOG_REPLAY, pid=15}] wal.WALSplitter(352): Last flushed sequenceid for 25de72d74cf42cf025decee5a0d3cd6e: last_flushed_sequence_id: 12 store_sequence_id { family_name: "cf1" sequence_id: 12 } store_sequence_id { family_name: "cf2" sequence_id: 12 } 2024-11-24T08:29:55,028 DEBUG [RS_LOG_REPLAY_OPS-regionserver/30c28c82771d:0-0 {event_type=RS_LOG_REPLAY, pid=15}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:46165/user/jenkins/test-data/7507fec3-e300-7585-3a43-9c9d3ed5b09e/WALs/30c28c82771d,38023,1732436977716-splitting/30c28c82771d%2C38023%2C1732436977716.1732436979045 so closing down 2024-11-24T08:29:55,028 DEBUG [RS_LOG_REPLAY_OPS-regionserver/30c28c82771d:0-0 {event_type=RS_LOG_REPLAY, pid=15}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-24T08:29:55,028 INFO [RS_LOG_REPLAY_OPS-regionserver/30c28c82771d:0-0 {event_type=RS_LOG_REPLAY, pid=15}] wal.OutputSink(145): 3 split writer threads finished 2024-11-24T08:29:55,028 INFO [RS_LOG_REPLAY_OPS-regionserver/30c28c82771d:0-0 {event_type=RS_LOG_REPLAY, pid=15}] wal.WALSplitter(425): Processed 6 edits across 0 Regions in 8 ms; skipped=6; WAL=hdfs://localhost:46165/user/jenkins/test-data/7507fec3-e300-7585-3a43-9c9d3ed5b09e/WALs/30c28c82771d,38023,1732436977716-splitting/30c28c82771d%2C38023%2C1732436977716.1732436979045, size=1.6 K, length=1677, corrupted=false, cancelled=false 2024-11-24T08:29:55,028 DEBUG [RS_LOG_REPLAY_OPS-regionserver/30c28c82771d:0-0 {event_type=RS_LOG_REPLAY, pid=15}] wal.WALSplitter(428): Completed split of hdfs://localhost:46165/user/jenkins/test-data/7507fec3-e300-7585-3a43-9c9d3ed5b09e/WALs/30c28c82771d,38023,1732436977716-splitting/30c28c82771d%2C38023%2C1732436977716.1732436979045, journal: Splitting hdfs://localhost:46165/user/jenkins/test-data/7507fec3-e300-7585-3a43-9c9d3ed5b09e/WALs/30c28c82771d,38023,1732436977716-splitting/30c28c82771d%2C38023%2C1732436977716.1732436979045, size=1.6 K (1677bytes) at 1732436995015Finishing writing output for hdfs://localhost:46165/user/jenkins/test-data/7507fec3-e300-7585-3a43-9c9d3ed5b09e/WALs/30c28c82771d,38023,1732436977716-splitting/30c28c82771d%2C38023%2C1732436977716.1732436979045 so closing down at 1732436995028 (+13 ms)3 split writer threads finished at 1732436995028Processed 6 edits across 0 Regions in 8 ms; skipped=6; WAL=hdfs://localhost:46165/user/jenkins/test-data/7507fec3-e300-7585-3a43-9c9d3ed5b09e/WALs/30c28c82771d,38023,1732436977716-splitting/30c28c82771d%2C38023%2C1732436977716.1732436979045, size=1.6 K, length=1677, corrupted=false, cancelled=false at 1732436995028 2024-11-24T08:29:55,029 DEBUG [RS_LOG_REPLAY_OPS-regionserver/30c28c82771d:0-0 {event_type=RS_LOG_REPLAY, pid=15}] regionserver.SplitLogWorker(218): Done splitting WAL hdfs://localhost:46165/user/jenkins/test-data/7507fec3-e300-7585-3a43-9c9d3ed5b09e/WALs/30c28c82771d,38023,1732436977716-splitting/30c28c82771d%2C38023%2C1732436977716.1732436979045 2024-11-24T08:29:55,030 DEBUG [RS_LOG_REPLAY_OPS-regionserver/30c28c82771d:0-0 {event_type=RS_LOG_REPLAY, pid=15}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=15 2024-11-24T08:29:55,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37719 {}] master.HMaster(4169): Remote procedure done, pid=15 2024-11-24T08:29:55,037 INFO [PEWorker-4 {}] wal.WALSplitUtil(143): Moved hdfs://localhost:46165/user/jenkins/test-data/7507fec3-e300-7585-3a43-9c9d3ed5b09e/WALs/30c28c82771d,38023,1732436977716-splitting/30c28c82771d%2C38023%2C1732436977716.1732436979045 to hdfs://localhost:46165/user/jenkins/test-data/7507fec3-e300-7585-3a43-9c9d3ed5b09e/oldWALs 2024-11-24T08:29:55,039 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=15, resume processing ppid=14 2024-11-24T08:29:55,039 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=15, ppid=14, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 30c28c82771d%2C38023%2C1732436977716.1732436979045, worker=30c28c82771d,40241,1732436977670 in 202 msec 2024-11-24T08:29:55,041 DEBUG [PEWorker-5 {}] master.SplitWALManager(172): Release split WAL worker=30c28c82771d,40241,1732436977670 2024-11-24T08:29:55,044 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=14, resume processing ppid=13 2024-11-24T08:29:55,045 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=14, ppid=13, state=SUCCESS, hasLock=false; SplitWALProcedure 30c28c82771d%2C38023%2C1732436977716.1732436979045, worker=30c28c82771d,40241,1732436977670 in 214 msec 2024-11-24T08:29:55,047 INFO [PEWorker-1 {}] master.SplitLogManager(171): hdfs://localhost:46165/user/jenkins/test-data/7507fec3-e300-7585-3a43-9c9d3ed5b09e/WALs/30c28c82771d,38023,1732436977716-splitting dir is empty, no logs to split. 2024-11-24T08:29:55,047 INFO [PEWorker-1 {}] master.SplitWALManager(105): 30c28c82771d,38023,1732436977716 WAL count=0, meta=false 2024-11-24T08:29:55,047 DEBUG [PEWorker-1 {}] procedure.ServerCrashProcedure(329): Check if 30c28c82771d,38023,1732436977716 WAL splitting is done? wals=0, meta=false 2024-11-24T08:29:55,050 INFO [PEWorker-1 {}] procedure.ServerCrashProcedure(321): Remove WAL directory for 30c28c82771d,38023,1732436977716 failed, ignore...File hdfs://localhost:46165/user/jenkins/test-data/7507fec3-e300-7585-3a43-9c9d3ed5b09e/WALs/30c28c82771d,38023,1732436977716-splitting does not exist. 2024-11-24T08:29:55,054 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=16, ppid=13, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=25de72d74cf42cf025decee5a0d3cd6e, ASSIGN}] 2024-11-24T08:29:55,056 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=16, ppid=13, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=25de72d74cf42cf025decee5a0d3cd6e, ASSIGN 2024-11-24T08:29:55,058 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=16, ppid=13, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=25de72d74cf42cf025decee5a0d3cd6e, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-11-24T08:29:55,208 DEBUG [30c28c82771d:37719 {}] balancer.BalancerClusterState(204): Hosts are {30c28c82771d=0} racks are {/default-rack=0} 2024-11-24T08:29:55,209 DEBUG [30c28c82771d:37719 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-11-24T08:29:55,209 DEBUG [30c28c82771d:37719 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-11-24T08:29:55,209 DEBUG [30c28c82771d:37719 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-11-24T08:29:55,209 DEBUG [30c28c82771d:37719 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-11-24T08:29:55,209 INFO [30c28c82771d:37719 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-11-24T08:29:55,209 INFO [30c28c82771d:37719 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-11-24T08:29:55,209 DEBUG [30c28c82771d:37719 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-11-24T08:29:55,209 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=16 updating hbase:meta row=25de72d74cf42cf025decee5a0d3cd6e, regionState=OPENING, regionLocation=30c28c82771d,40241,1732436977670 2024-11-24T08:29:55,212 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=16, ppid=13, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=25de72d74cf42cf025decee5a0d3cd6e, ASSIGN because future has completed 2024-11-24T08:29:55,213 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=17, ppid=16, state=RUNNABLE, hasLock=false; OpenRegionProcedure 25de72d74cf42cf025decee5a0d3cd6e, server=30c28c82771d,40241,1732436977670}] 2024-11-24T08:29:55,227 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testReplayEditsAfterRegionMovedWithMultiCF', row='r1', locateType=CURRENT is [region=testReplayEditsAfterRegionMovedWithMultiCF,,1732436992313.25de72d74cf42cf025decee5a0d3cd6e., hostname=30c28c82771d,38023,1732436977716, seqNum=18] 2024-11-24T08:29:55,228 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] ipc.AbstractRpcClient(357): Not trying to connect to 30c28c82771d:38023 this server is in the failed servers list 2024-11-24T08:29:55,228 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1732436992313.25de72d74cf42cf025decee5a0d3cd6e., hostname=30c28c82771d,38023,1732436977716, seqNum=18 , the old value is region=testReplayEditsAfterRegionMovedWithMultiCF,,1732436992313.25de72d74cf42cf025decee5a0d3cd6e., hostname=30c28c82771d,38023,1732436977716, seqNum=18, error=org.apache.hadoop.hbase.ipc.FailedServerException: Call to address=30c28c82771d:38023 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: 30c28c82771d:38023 2024-11-24T08:29:55,228 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1732436992313.25de72d74cf42cf025decee5a0d3cd6e., hostname=30c28c82771d,38023,1732436977716, seqNum=18 is org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: 30c28c82771d:38023 2024-11-24T08:29:55,228 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncRegionLocatorHelper(88): Try removing region=testReplayEditsAfterRegionMovedWithMultiCF,,1732436992313.25de72d74cf42cf025decee5a0d3cd6e., hostname=30c28c82771d,38023,1732436977716, seqNum=18 from cache 2024-11-24T08:29:55,369 INFO [RS_OPEN_REGION-regionserver/30c28c82771d:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] handler.AssignRegionHandler(132): Open testReplayEditsAfterRegionMovedWithMultiCF,,1732436992313.25de72d74cf42cf025decee5a0d3cd6e. 2024-11-24T08:29:55,370 DEBUG [RS_OPEN_REGION-regionserver/30c28c82771d:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegion(7752): Opening region: {ENCODED => 25de72d74cf42cf025decee5a0d3cd6e, NAME => 'testReplayEditsAfterRegionMovedWithMultiCF,,1732436992313.25de72d74cf42cf025decee5a0d3cd6e.', STARTKEY => '', ENDKEY => ''} 2024-11-24T08:29:55,370 DEBUG [RS_OPEN_REGION-regionserver/30c28c82771d:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testReplayEditsAfterRegionMovedWithMultiCF 25de72d74cf42cf025decee5a0d3cd6e 2024-11-24T08:29:55,370 DEBUG [RS_OPEN_REGION-regionserver/30c28c82771d:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegion(898): Instantiated testReplayEditsAfterRegionMovedWithMultiCF,,1732436992313.25de72d74cf42cf025decee5a0d3cd6e.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T08:29:55,370 DEBUG [RS_OPEN_REGION-regionserver/30c28c82771d:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegion(7794): checking encryption for 25de72d74cf42cf025decee5a0d3cd6e 2024-11-24T08:29:55,370 DEBUG [RS_OPEN_REGION-regionserver/30c28c82771d:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegion(7797): checking classloading for 25de72d74cf42cf025decee5a0d3cd6e 2024-11-24T08:29:55,385 INFO [StoreOpener-25de72d74cf42cf025decee5a0d3cd6e-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf1 of region 25de72d74cf42cf025decee5a0d3cd6e 2024-11-24T08:29:55,386 INFO [StoreOpener-25de72d74cf42cf025decee5a0d3cd6e-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 25de72d74cf42cf025decee5a0d3cd6e columnFamilyName cf1 2024-11-24T08:29:55,386 DEBUG [StoreOpener-25de72d74cf42cf025decee5a0d3cd6e-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:29:55,394 DEBUG [StoreOpener-25de72d74cf42cf025decee5a0d3cd6e-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:46165/user/jenkins/test-data/7507fec3-e300-7585-3a43-9c9d3ed5b09e/data/default/testReplayEditsAfterRegionMovedWithMultiCF/25de72d74cf42cf025decee5a0d3cd6e/cf1/60c7344f8e564ddf862b0c7395425cee 2024-11-24T08:29:55,394 INFO [StoreOpener-25de72d74cf42cf025decee5a0d3cd6e-1 {}] regionserver.HStore(327): Store=25de72d74cf42cf025decee5a0d3cd6e/cf1, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T08:29:55,394 INFO [StoreOpener-25de72d74cf42cf025decee5a0d3cd6e-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf2 of region 25de72d74cf42cf025decee5a0d3cd6e 2024-11-24T08:29:55,400 INFO [StoreOpener-25de72d74cf42cf025decee5a0d3cd6e-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 25de72d74cf42cf025decee5a0d3cd6e columnFamilyName cf2 2024-11-24T08:29:55,400 DEBUG [StoreOpener-25de72d74cf42cf025decee5a0d3cd6e-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:29:55,406 DEBUG [StoreOpener-25de72d74cf42cf025decee5a0d3cd6e-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:46165/user/jenkins/test-data/7507fec3-e300-7585-3a43-9c9d3ed5b09e/data/default/testReplayEditsAfterRegionMovedWithMultiCF/25de72d74cf42cf025decee5a0d3cd6e/cf2/84a6767619e642f88ded3e4c594f638d 2024-11-24T08:29:55,406 INFO [StoreOpener-25de72d74cf42cf025decee5a0d3cd6e-1 {}] regionserver.HStore(327): Store=25de72d74cf42cf025decee5a0d3cd6e/cf2, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T08:29:55,407 DEBUG [RS_OPEN_REGION-regionserver/30c28c82771d:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegion(1038): replaying wal for 25de72d74cf42cf025decee5a0d3cd6e 2024-11-24T08:29:55,407 DEBUG [RS_OPEN_REGION-regionserver/30c28c82771d:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46165/user/jenkins/test-data/7507fec3-e300-7585-3a43-9c9d3ed5b09e/data/default/testReplayEditsAfterRegionMovedWithMultiCF/25de72d74cf42cf025decee5a0d3cd6e 2024-11-24T08:29:55,409 DEBUG [RS_OPEN_REGION-regionserver/30c28c82771d:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46165/user/jenkins/test-data/7507fec3-e300-7585-3a43-9c9d3ed5b09e/data/default/testReplayEditsAfterRegionMovedWithMultiCF/25de72d74cf42cf025decee5a0d3cd6e 2024-11-24T08:29:55,410 DEBUG [RS_OPEN_REGION-regionserver/30c28c82771d:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegion(1048): stopping wal replay for 25de72d74cf42cf025decee5a0d3cd6e 2024-11-24T08:29:55,410 DEBUG [RS_OPEN_REGION-regionserver/30c28c82771d:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegion(1060): Cleaning up temporary data for 25de72d74cf42cf025decee5a0d3cd6e 2024-11-24T08:29:55,411 DEBUG [RS_OPEN_REGION-regionserver/30c28c82771d:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsAfterRegionMovedWithMultiCF descriptor;using region.getMemStoreFlushHeapSize/# of families (64.0 M)) instead. 2024-11-24T08:29:55,414 DEBUG [RS_OPEN_REGION-regionserver/30c28c82771d:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegion(1093): writing seq id for 25de72d74cf42cf025decee5a0d3cd6e 2024-11-24T08:29:55,415 INFO [RS_OPEN_REGION-regionserver/30c28c82771d:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegion(1114): Opened 25de72d74cf42cf025decee5a0d3cd6e; next sequenceid=18; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=70707125, jitterRate=0.05361826717853546}}}, FlushLargeStoresPolicy{flushSizeLowerBound=67108864} 2024-11-24T08:29:55,415 DEBUG [RS_OPEN_REGION-regionserver/30c28c82771d:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 25de72d74cf42cf025decee5a0d3cd6e 2024-11-24T08:29:55,415 DEBUG [RS_OPEN_REGION-regionserver/30c28c82771d:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegion(1006): Region open journal for 25de72d74cf42cf025decee5a0d3cd6e: Running coprocessor pre-open hook at 1732436995370Writing region info on filesystem at 1732436995370Initializing all the Stores at 1732436995371 (+1 ms)Instantiating store for column family {NAME => 'cf1', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732436995371Instantiating store for column family {NAME => 'cf2', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732436995384 (+13 ms)Cleaning up temporary data from old regions at 1732436995410 (+26 ms)Running coprocessor post-open hooks at 1732436995415 (+5 ms)Region opened successfully at 1732436995415 2024-11-24T08:29:55,417 INFO [RS_OPEN_REGION-regionserver/30c28c82771d:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegionServer(2236): Post open deploy tasks for testReplayEditsAfterRegionMovedWithMultiCF,,1732436992313.25de72d74cf42cf025decee5a0d3cd6e., pid=17, masterSystemTime=1732436995365 2024-11-24T08:29:55,419 DEBUG [RS_OPEN_REGION-regionserver/30c28c82771d:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegionServer(2266): Finished post open deploy task for testReplayEditsAfterRegionMovedWithMultiCF,,1732436992313.25de72d74cf42cf025decee5a0d3cd6e. 2024-11-24T08:29:55,419 INFO [RS_OPEN_REGION-regionserver/30c28c82771d:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] handler.AssignRegionHandler(153): Opened testReplayEditsAfterRegionMovedWithMultiCF,,1732436992313.25de72d74cf42cf025decee5a0d3cd6e. 2024-11-24T08:29:55,420 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=16 updating hbase:meta row=25de72d74cf42cf025decee5a0d3cd6e, regionState=OPEN, openSeqNum=18, regionLocation=30c28c82771d,40241,1732436977670 2024-11-24T08:29:55,423 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=17, ppid=16, state=RUNNABLE, hasLock=false; OpenRegionProcedure 25de72d74cf42cf025decee5a0d3cd6e, server=30c28c82771d,40241,1732436977670 because future has completed 2024-11-24T08:29:55,428 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=17, resume processing ppid=16 2024-11-24T08:29:55,428 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=17, ppid=16, state=SUCCESS, hasLock=false; OpenRegionProcedure 25de72d74cf42cf025decee5a0d3cd6e, server=30c28c82771d,40241,1732436977670 in 213 msec 2024-11-24T08:29:55,431 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=16, resume processing ppid=13 2024-11-24T08:29:55,431 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=16, ppid=13, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=25de72d74cf42cf025decee5a0d3cd6e, ASSIGN in 374 msec 2024-11-24T08:29:55,431 INFO [PEWorker-4 {}] procedure.ServerCrashProcedure(291): removed crashed server 30c28c82771d,38023,1732436977716 after splitting done 2024-11-24T08:29:55,434 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=13, state=SUCCESS, hasLock=false; ServerCrashProcedure 30c28c82771d,38023,1732436977716, splitWal=true, meta=false in 620 msec 2024-11-24T08:29:55,737 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testReplayEditsAfterRegionMovedWithMultiCF', row='r1', locateType=CURRENT is [region=testReplayEditsAfterRegionMovedWithMultiCF,,1732436992313.25de72d74cf42cf025decee5a0d3cd6e., hostname=30c28c82771d,40241,1732436977670, seqNum=18] 2024-11-24T08:29:55,754 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestAsyncWALReplayCompressed#testReplayEditsAfterRegionMovedWithMultiCF Thread=408 (was 409), OpenFileDescriptor=1051 (was 1023) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=169 (was 149) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=6674 (was 6698) 2024-11-24T08:29:55,755 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1051 is superior to 1024 2024-11-24T08:29:55,768 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestAsyncWALReplayCompressed#testReplayEditsAfterPartialFlush Thread=408, OpenFileDescriptor=1051, MaxFileDescriptor=1048576, SystemLoadAverage=169, ProcessCount=11, AvailableMemoryMB=6673 2024-11-24T08:29:55,768 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1051 is superior to 1024 2024-11-24T08:29:55,783 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-24T08:29:55,785 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-24T08:29:55,785 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-24T08:29:55,788 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-69506817, suffix=, logDir=hdfs://localhost:46165/hbase/WALs/hregion-69506817, archiveDir=hdfs://localhost:46165/hbase/oldWALs, maxLogs=32 2024-11-24T08:29:55,801 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-69506817/hregion-69506817.1732436995788, exclude list is [], retry=0 2024-11-24T08:29:55,803 DEBUG [AsyncFSWAL-20-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40533,DS-5eeb9666-6dd5-4a48-9fb1-3f9aff1acdca,DISK] 2024-11-24T08:29:55,803 DEBUG [AsyncFSWAL-20-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33555,DS-8d97c3e2-1525-418c-a970-00a7157a0d31,DISK] 2024-11-24T08:29:55,804 DEBUG [AsyncFSWAL-20-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:34459,DS-6b4d1656-f49b-4a16-be49-2eb682c16e11,DISK] 2024-11-24T08:29:55,806 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-69506817/hregion-69506817.1732436995788 2024-11-24T08:29:55,806 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:42673:42673),(127.0.0.1/127.0.0.1:39357:39357),(127.0.0.1/127.0.0.1:44131:44131)] 2024-11-24T08:29:55,807 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => 6ec96c2b8dccfd02b257c734dcf0524d, NAME => 'testReplayEditsWrittenViaHRegion,,1732436995784.6ec96c2b8dccfd02b257c734dcf0524d.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testReplayEditsWrittenViaHRegion', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:46165/hbase 2024-11-24T08:29:55,815 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33555 is added to blk_1073741893_1071 (size=67) 2024-11-24T08:29:55,816 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34459 is added to blk_1073741893_1071 (size=67) 2024-11-24T08:29:55,816 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40533 is added to blk_1073741893_1071 (size=67) 2024-11-24T08:29:55,817 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenViaHRegion,,1732436995784.6ec96c2b8dccfd02b257c734dcf0524d.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T08:29:55,819 INFO [StoreOpener-6ec96c2b8dccfd02b257c734dcf0524d-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 6ec96c2b8dccfd02b257c734dcf0524d 2024-11-24T08:29:55,820 INFO [StoreOpener-6ec96c2b8dccfd02b257c734dcf0524d-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 6ec96c2b8dccfd02b257c734dcf0524d columnFamilyName a 2024-11-24T08:29:55,820 DEBUG [StoreOpener-6ec96c2b8dccfd02b257c734dcf0524d-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:29:55,821 INFO [StoreOpener-6ec96c2b8dccfd02b257c734dcf0524d-1 {}] regionserver.HStore(327): Store=6ec96c2b8dccfd02b257c734dcf0524d/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T08:29:55,821 INFO [StoreOpener-6ec96c2b8dccfd02b257c734dcf0524d-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 6ec96c2b8dccfd02b257c734dcf0524d 2024-11-24T08:29:55,825 INFO [StoreOpener-6ec96c2b8dccfd02b257c734dcf0524d-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 6ec96c2b8dccfd02b257c734dcf0524d columnFamilyName b 2024-11-24T08:29:55,825 DEBUG [StoreOpener-6ec96c2b8dccfd02b257c734dcf0524d-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:29:55,825 INFO [StoreOpener-6ec96c2b8dccfd02b257c734dcf0524d-1 {}] regionserver.HStore(327): Store=6ec96c2b8dccfd02b257c734dcf0524d/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T08:29:55,825 INFO [StoreOpener-6ec96c2b8dccfd02b257c734dcf0524d-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 6ec96c2b8dccfd02b257c734dcf0524d 2024-11-24T08:29:55,827 INFO [StoreOpener-6ec96c2b8dccfd02b257c734dcf0524d-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 6ec96c2b8dccfd02b257c734dcf0524d columnFamilyName c 2024-11-24T08:29:55,827 DEBUG [StoreOpener-6ec96c2b8dccfd02b257c734dcf0524d-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:29:55,828 INFO [StoreOpener-6ec96c2b8dccfd02b257c734dcf0524d-1 {}] regionserver.HStore(327): Store=6ec96c2b8dccfd02b257c734dcf0524d/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T08:29:55,828 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 6ec96c2b8dccfd02b257c734dcf0524d 2024-11-24T08:29:55,829 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46165/hbase/data/default/testReplayEditsWrittenViaHRegion/6ec96c2b8dccfd02b257c734dcf0524d 2024-11-24T08:29:55,829 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46165/hbase/data/default/testReplayEditsWrittenViaHRegion/6ec96c2b8dccfd02b257c734dcf0524d 2024-11-24T08:29:55,831 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 6ec96c2b8dccfd02b257c734dcf0524d 2024-11-24T08:29:55,831 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 6ec96c2b8dccfd02b257c734dcf0524d 2024-11-24T08:29:55,831 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsWrittenViaHRegion descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-24T08:29:55,832 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 6ec96c2b8dccfd02b257c734dcf0524d 2024-11-24T08:29:55,835 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:46165/hbase/data/default/testReplayEditsWrittenViaHRegion/6ec96c2b8dccfd02b257c734dcf0524d/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-24T08:29:55,835 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 6ec96c2b8dccfd02b257c734dcf0524d; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=73767948, jitterRate=0.09922808408737183}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-24T08:29:55,836 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 6ec96c2b8dccfd02b257c734dcf0524d: Writing region info on filesystem at 1732436995817Initializing all the Stores at 1732436995818 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732436995818Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732436995818Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732436995819 (+1 ms)Cleaning up temporary data from old regions at 1732436995831 (+12 ms)Region opened successfully at 1732436995836 (+5 ms) 2024-11-24T08:29:55,836 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 6ec96c2b8dccfd02b257c734dcf0524d, disabling compactions & flushes 2024-11-24T08:29:55,836 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsWrittenViaHRegion,,1732436995784.6ec96c2b8dccfd02b257c734dcf0524d. 2024-11-24T08:29:55,836 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsWrittenViaHRegion,,1732436995784.6ec96c2b8dccfd02b257c734dcf0524d. 2024-11-24T08:29:55,836 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsWrittenViaHRegion,,1732436995784.6ec96c2b8dccfd02b257c734dcf0524d. after waiting 0 ms 2024-11-24T08:29:55,836 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsWrittenViaHRegion,,1732436995784.6ec96c2b8dccfd02b257c734dcf0524d. 2024-11-24T08:29:55,836 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsWrittenViaHRegion,,1732436995784.6ec96c2b8dccfd02b257c734dcf0524d. 2024-11-24T08:29:55,836 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 6ec96c2b8dccfd02b257c734dcf0524d: Waiting for close lock at 1732436995836Disabling compacts and flushes for region at 1732436995836Disabling writes for close at 1732436995836Writing region close event to WAL at 1732436995836Closed at 1732436995836 2024-11-24T08:29:55,840 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33555 is added to blk_1073741892_1070 (size=93) 2024-11-24T08:29:55,845 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34459 is added to blk_1073741892_1070 (size=93) 2024-11-24T08:29:55,845 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40533 is added to blk_1073741892_1070 (size=93) 2024-11-24T08:29:55,856 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-11-24T08:29:55,856 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-69506817:(num 1732436995788) 2024-11-24T08:29:55,856 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-24T08:29:55,858 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:46165/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1732436995782, archiveDir=hdfs://localhost:46165/hbase/oldWALs, maxLogs=32 2024-11-24T08:29:55,876 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1732436995782/wal.1732436995859, exclude list is [], retry=0 2024-11-24T08:29:55,879 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33555,DS-8d97c3e2-1525-418c-a970-00a7157a0d31,DISK] 2024-11-24T08:29:55,880 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40533,DS-5eeb9666-6dd5-4a48-9fb1-3f9aff1acdca,DISK] 2024-11-24T08:29:55,880 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:34459,DS-6b4d1656-f49b-4a16-be49-2eb682c16e11,DISK] 2024-11-24T08:29:55,895 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1732436995782/wal.1732436995859 2024-11-24T08:29:55,895 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:39357:39357),(127.0.0.1/127.0.0.1:42673:42673),(127.0.0.1/127.0.0.1:44131:44131)] 2024-11-24T08:29:55,895 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 6ec96c2b8dccfd02b257c734dcf0524d, NAME => 'testReplayEditsWrittenViaHRegion,,1732436995784.6ec96c2b8dccfd02b257c734dcf0524d.', STARTKEY => '', ENDKEY => ''} 2024-11-24T08:29:55,896 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenViaHRegion,,1732436995784.6ec96c2b8dccfd02b257c734dcf0524d.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T08:29:55,896 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 6ec96c2b8dccfd02b257c734dcf0524d 2024-11-24T08:29:55,896 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 6ec96c2b8dccfd02b257c734dcf0524d 2024-11-24T08:29:55,898 INFO [StoreOpener-6ec96c2b8dccfd02b257c734dcf0524d-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 6ec96c2b8dccfd02b257c734dcf0524d 2024-11-24T08:29:55,898 INFO [StoreOpener-6ec96c2b8dccfd02b257c734dcf0524d-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 6ec96c2b8dccfd02b257c734dcf0524d columnFamilyName a 2024-11-24T08:29:55,899 DEBUG [StoreOpener-6ec96c2b8dccfd02b257c734dcf0524d-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:29:55,899 INFO [StoreOpener-6ec96c2b8dccfd02b257c734dcf0524d-1 {}] regionserver.HStore(327): Store=6ec96c2b8dccfd02b257c734dcf0524d/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T08:29:55,899 INFO [StoreOpener-6ec96c2b8dccfd02b257c734dcf0524d-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 6ec96c2b8dccfd02b257c734dcf0524d 2024-11-24T08:29:55,900 INFO [StoreOpener-6ec96c2b8dccfd02b257c734dcf0524d-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 6ec96c2b8dccfd02b257c734dcf0524d columnFamilyName b 2024-11-24T08:29:55,900 DEBUG [StoreOpener-6ec96c2b8dccfd02b257c734dcf0524d-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:29:55,901 INFO [StoreOpener-6ec96c2b8dccfd02b257c734dcf0524d-1 {}] regionserver.HStore(327): Store=6ec96c2b8dccfd02b257c734dcf0524d/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T08:29:55,901 INFO [StoreOpener-6ec96c2b8dccfd02b257c734dcf0524d-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 6ec96c2b8dccfd02b257c734dcf0524d 2024-11-24T08:29:55,902 INFO [StoreOpener-6ec96c2b8dccfd02b257c734dcf0524d-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 6ec96c2b8dccfd02b257c734dcf0524d columnFamilyName c 2024-11-24T08:29:55,902 DEBUG [StoreOpener-6ec96c2b8dccfd02b257c734dcf0524d-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:29:55,904 INFO [StoreOpener-6ec96c2b8dccfd02b257c734dcf0524d-1 {}] regionserver.HStore(327): Store=6ec96c2b8dccfd02b257c734dcf0524d/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T08:29:55,904 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 6ec96c2b8dccfd02b257c734dcf0524d 2024-11-24T08:29:55,905 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46165/hbase/data/default/testReplayEditsWrittenViaHRegion/6ec96c2b8dccfd02b257c734dcf0524d 2024-11-24T08:29:55,906 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46165/hbase/data/default/testReplayEditsWrittenViaHRegion/6ec96c2b8dccfd02b257c734dcf0524d 2024-11-24T08:29:55,908 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 6ec96c2b8dccfd02b257c734dcf0524d 2024-11-24T08:29:55,908 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 6ec96c2b8dccfd02b257c734dcf0524d 2024-11-24T08:29:55,908 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsWrittenViaHRegion descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-24T08:29:55,913 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 6ec96c2b8dccfd02b257c734dcf0524d 2024-11-24T08:29:55,915 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 6ec96c2b8dccfd02b257c734dcf0524d; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=64948824, jitterRate=-0.03218710422515869}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-24T08:29:55,916 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 6ec96c2b8dccfd02b257c734dcf0524d: Writing region info on filesystem at 1732436995896Initializing all the Stores at 1732436995897 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732436995897Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732436995897Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732436995897Cleaning up temporary data from old regions at 1732436995908 (+11 ms)Region opened successfully at 1732436995916 (+8 ms) 2024-11-24T08:29:55,962 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 6ec96c2b8dccfd02b257c734dcf0524d 3/3 column families, dataSize=2.55 KB heapSize=5.44 KB 2024-11-24T08:29:55,995 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46165/hbase/data/default/testReplayEditsWrittenViaHRegion/6ec96c2b8dccfd02b257c734dcf0524d/.tmp/a/c19e76f417474e619eb423745b198822 is 91, key is testReplayEditsWrittenViaHRegion/a:x0/1732436995916/Put/seqid=0 2024-11-24T08:29:56,006 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33555 is added to blk_1073741895_1073 (size=5958) 2024-11-24T08:29:56,006 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34459 is added to blk_1073741895_1073 (size=5958) 2024-11-24T08:29:56,007 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40533 is added to blk_1073741895_1073 (size=5958) 2024-11-24T08:29:56,007 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=33 (bloomFilter=true), to=hdfs://localhost:46165/hbase/data/default/testReplayEditsWrittenViaHRegion/6ec96c2b8dccfd02b257c734dcf0524d/.tmp/a/c19e76f417474e619eb423745b198822 2024-11-24T08:29:56,031 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46165/hbase/data/default/testReplayEditsWrittenViaHRegion/6ec96c2b8dccfd02b257c734dcf0524d/.tmp/b/9a25ce57ceeb49ea8652ec2937a7de3b is 91, key is testReplayEditsWrittenViaHRegion/b:x0/1732436995928/Put/seqid=0 2024-11-24T08:29:56,038 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33555 is added to blk_1073741896_1074 (size=5958) 2024-11-24T08:29:56,038 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34459 is added to blk_1073741896_1074 (size=5958) 2024-11-24T08:29:56,039 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40533 is added to blk_1073741896_1074 (size=5958) 2024-11-24T08:29:56,039 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=33 (bloomFilter=true), to=hdfs://localhost:46165/hbase/data/default/testReplayEditsWrittenViaHRegion/6ec96c2b8dccfd02b257c734dcf0524d/.tmp/b/9a25ce57ceeb49ea8652ec2937a7de3b 2024-11-24T08:29:56,067 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46165/hbase/data/default/testReplayEditsWrittenViaHRegion/6ec96c2b8dccfd02b257c734dcf0524d/.tmp/c/9c44fbe8921d40908e624e046d28f719 is 91, key is testReplayEditsWrittenViaHRegion/c:x0/1732436995946/Put/seqid=0 2024-11-24T08:29:56,075 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33555 is added to blk_1073741897_1075 (size=5958) 2024-11-24T08:29:56,075 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34459 is added to blk_1073741897_1075 (size=5958) 2024-11-24T08:29:56,075 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40533 is added to blk_1073741897_1075 (size=5958) 2024-11-24T08:29:56,076 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=33 (bloomFilter=true), to=hdfs://localhost:46165/hbase/data/default/testReplayEditsWrittenViaHRegion/6ec96c2b8dccfd02b257c734dcf0524d/.tmp/c/9c44fbe8921d40908e624e046d28f719 2024-11-24T08:29:56,084 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46165/hbase/data/default/testReplayEditsWrittenViaHRegion/6ec96c2b8dccfd02b257c734dcf0524d/.tmp/a/c19e76f417474e619eb423745b198822 as hdfs://localhost:46165/hbase/data/default/testReplayEditsWrittenViaHRegion/6ec96c2b8dccfd02b257c734dcf0524d/a/c19e76f417474e619eb423745b198822 2024-11-24T08:29:56,090 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46165/hbase/data/default/testReplayEditsWrittenViaHRegion/6ec96c2b8dccfd02b257c734dcf0524d/a/c19e76f417474e619eb423745b198822, entries=10, sequenceid=33, filesize=5.8 K 2024-11-24T08:29:56,091 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46165/hbase/data/default/testReplayEditsWrittenViaHRegion/6ec96c2b8dccfd02b257c734dcf0524d/.tmp/b/9a25ce57ceeb49ea8652ec2937a7de3b as hdfs://localhost:46165/hbase/data/default/testReplayEditsWrittenViaHRegion/6ec96c2b8dccfd02b257c734dcf0524d/b/9a25ce57ceeb49ea8652ec2937a7de3b 2024-11-24T08:29:56,097 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46165/hbase/data/default/testReplayEditsWrittenViaHRegion/6ec96c2b8dccfd02b257c734dcf0524d/b/9a25ce57ceeb49ea8652ec2937a7de3b, entries=10, sequenceid=33, filesize=5.8 K 2024-11-24T08:29:56,098 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46165/hbase/data/default/testReplayEditsWrittenViaHRegion/6ec96c2b8dccfd02b257c734dcf0524d/.tmp/c/9c44fbe8921d40908e624e046d28f719 as hdfs://localhost:46165/hbase/data/default/testReplayEditsWrittenViaHRegion/6ec96c2b8dccfd02b257c734dcf0524d/c/9c44fbe8921d40908e624e046d28f719 2024-11-24T08:29:56,103 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46165/hbase/data/default/testReplayEditsWrittenViaHRegion/6ec96c2b8dccfd02b257c734dcf0524d/c/9c44fbe8921d40908e624e046d28f719, entries=10, sequenceid=33, filesize=5.8 K 2024-11-24T08:29:56,105 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~2.55 KB/2610, heapSize ~5.39 KB/5520, currentSize=0 B/0 for 6ec96c2b8dccfd02b257c734dcf0524d in 142ms, sequenceid=33, compaction requested=false 2024-11-24T08:29:56,105 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for 6ec96c2b8dccfd02b257c734dcf0524d: 2024-11-24T08:29:56,105 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 6ec96c2b8dccfd02b257c734dcf0524d, disabling compactions & flushes 2024-11-24T08:29:56,105 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsWrittenViaHRegion,,1732436995784.6ec96c2b8dccfd02b257c734dcf0524d. 2024-11-24T08:29:56,105 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsWrittenViaHRegion,,1732436995784.6ec96c2b8dccfd02b257c734dcf0524d. 2024-11-24T08:29:56,105 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsWrittenViaHRegion,,1732436995784.6ec96c2b8dccfd02b257c734dcf0524d. after waiting 0 ms 2024-11-24T08:29:56,105 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsWrittenViaHRegion,,1732436995784.6ec96c2b8dccfd02b257c734dcf0524d. 2024-11-24T08:29:56,111 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsWrittenViaHRegion,,1732436995784.6ec96c2b8dccfd02b257c734dcf0524d. 2024-11-24T08:29:56,111 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 6ec96c2b8dccfd02b257c734dcf0524d: Waiting for close lock at 1732436996105Disabling compacts and flushes for region at 1732436996105Disabling writes for close at 1732436996105Writing region close event to WAL at 1732436996110 (+5 ms)Closed at 1732436996111 (+1 ms) 2024-11-24T08:29:56,115 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33555 is added to blk_1073741894_1072 (size=2873) 2024-11-24T08:29:56,115 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34459 is added to blk_1073741894_1072 (size=2873) 2024-11-24T08:29:56,115 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40533 is added to blk_1073741894_1072 (size=2873) 2024-11-24T08:29:56,121 DEBUG [Time-limited test {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46165/hbase/data/default/testReplayEditsWrittenViaHRegion/6ec96c2b8dccfd02b257c734dcf0524d/b/9a25ce57ceeb49ea8652ec2937a7de3b to hdfs://localhost:46165/hbase/archive/data/default/testReplayEditsWrittenViaHRegion/6ec96c2b8dccfd02b257c734dcf0524d/b/9a25ce57ceeb49ea8652ec2937a7de3b 2024-11-24T08:29:56,140 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:46165/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1732436995782/wal.1732436995859, size=2.8 K (2873bytes) 2024-11-24T08:29:56,140 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:46165/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1732436995782/wal.1732436995859 2024-11-24T08:29:56,140 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:46165/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1732436995782/wal.1732436995859 after 0ms 2024-11-24T08:29:56,143 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:46165/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1732436995782/wal.1732436995859: isRecoveredEdits=false, hasTagCompression=true, hasValueCompression=false, valueCompressionType=null 2024-11-24T08:29:56,143 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:46165/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1732436995782/wal.1732436995859 took 4ms 2024-11-24T08:29:56,158 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:46165/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1732436995782/wal.1732436995859 so closing down 2024-11-24T08:29:56,158 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-24T08:29:56,159 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000003-wal.1732436995859.temp 2024-11-24T08:29:56,164 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:46165/hbase/data/default/testReplayEditsWrittenViaHRegion/6ec96c2b8dccfd02b257c734dcf0524d/recovered.edits/0000000000000000003-wal.1732436995859.temp 2024-11-24T08:29:56,165 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-11-24T08:29:56,178 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34459 is added to blk_1073741898_1076 (size=2312) 2024-11-24T08:29:56,178 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33555 is added to blk_1073741898_1076 (size=2312) 2024-11-24T08:29:56,178 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40533 is added to blk_1073741898_1076 (size=2312) 2024-11-24T08:29:56,179 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:46165/hbase/data/default/testReplayEditsWrittenViaHRegion/6ec96c2b8dccfd02b257c734dcf0524d/recovered.edits/0000000000000000003-wal.1732436995859.temp (wrote 30 edits, skipped 0 edits in 0 ms) 2024-11-24T08:29:56,181 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:46165/hbase/data/default/testReplayEditsWrittenViaHRegion/6ec96c2b8dccfd02b257c734dcf0524d/recovered.edits/0000000000000000003-wal.1732436995859.temp to hdfs://localhost:46165/hbase/data/default/testReplayEditsWrittenViaHRegion/6ec96c2b8dccfd02b257c734dcf0524d/recovered.edits/0000000000000000032 2024-11-24T08:29:56,181 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 32 edits across 1 Regions in 25 ms; skipped=2; WAL=hdfs://localhost:46165/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1732436995782/wal.1732436995859, size=2.8 K, length=2873, corrupted=false, cancelled=false 2024-11-24T08:29:56,181 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:46165/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1732436995782/wal.1732436995859, journal: Splitting hdfs://localhost:46165/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1732436995782/wal.1732436995859, size=2.8 K (2873bytes) at 1732436996140Finishing writing output for hdfs://localhost:46165/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1732436995782/wal.1732436995859 so closing down at 1732436996158 (+18 ms)Creating recovered edits writer path=hdfs://localhost:46165/hbase/data/default/testReplayEditsWrittenViaHRegion/6ec96c2b8dccfd02b257c734dcf0524d/recovered.edits/0000000000000000003-wal.1732436995859.temp at 1732436996164 (+6 ms)3 split writer threads finished at 1732436996165 (+1 ms)Closed recovered edits writer path=hdfs://localhost:46165/hbase/data/default/testReplayEditsWrittenViaHRegion/6ec96c2b8dccfd02b257c734dcf0524d/recovered.edits/0000000000000000003-wal.1732436995859.temp (wrote 30 edits, skipped 0 edits in 0 ms) at 1732436996179 (+14 ms)Rename recovered edits hdfs://localhost:46165/hbase/data/default/testReplayEditsWrittenViaHRegion/6ec96c2b8dccfd02b257c734dcf0524d/recovered.edits/0000000000000000003-wal.1732436995859.temp to hdfs://localhost:46165/hbase/data/default/testReplayEditsWrittenViaHRegion/6ec96c2b8dccfd02b257c734dcf0524d/recovered.edits/0000000000000000032 at 1732436996181 (+2 ms)Processed 32 edits across 1 Regions in 25 ms; skipped=2; WAL=hdfs://localhost:46165/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1732436995782/wal.1732436995859, size=2.8 K, length=2873, corrupted=false, cancelled=false at 1732436996181 2024-11-24T08:29:56,183 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:46165/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1732436995782/wal.1732436995859 to hdfs://localhost:46165/hbase/oldWALs/wal.1732436995859 2024-11-24T08:29:56,184 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:46165/hbase/data/default/testReplayEditsWrittenViaHRegion/6ec96c2b8dccfd02b257c734dcf0524d/recovered.edits/0000000000000000032 2024-11-24T08:29:56,184 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-24T08:29:56,186 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:46165/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1732436995782, archiveDir=hdfs://localhost:46165/hbase/oldWALs, maxLogs=32 2024-11-24T08:29:56,199 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1732436995782/wal.1732436996186, exclude list is [], retry=0 2024-11-24T08:29:56,203 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40533,DS-5eeb9666-6dd5-4a48-9fb1-3f9aff1acdca,DISK] 2024-11-24T08:29:56,203 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:34459,DS-6b4d1656-f49b-4a16-be49-2eb682c16e11,DISK] 2024-11-24T08:29:56,203 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33555,DS-8d97c3e2-1525-418c-a970-00a7157a0d31,DISK] 2024-11-24T08:29:56,207 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1732436995782/wal.1732436996186 2024-11-24T08:29:56,207 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:42673:42673),(127.0.0.1/127.0.0.1:44131:44131),(127.0.0.1/127.0.0.1:39357:39357)] 2024-11-24T08:29:56,208 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 6ec96c2b8dccfd02b257c734dcf0524d, NAME => 'testReplayEditsWrittenViaHRegion,,1732436995784.6ec96c2b8dccfd02b257c734dcf0524d.', STARTKEY => '', ENDKEY => ''} 2024-11-24T08:29:56,208 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenViaHRegion,,1732436995784.6ec96c2b8dccfd02b257c734dcf0524d.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T08:29:56,208 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 6ec96c2b8dccfd02b257c734dcf0524d 2024-11-24T08:29:56,208 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 6ec96c2b8dccfd02b257c734dcf0524d 2024-11-24T08:29:56,209 INFO [StoreOpener-6ec96c2b8dccfd02b257c734dcf0524d-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 6ec96c2b8dccfd02b257c734dcf0524d 2024-11-24T08:29:56,210 INFO [StoreOpener-6ec96c2b8dccfd02b257c734dcf0524d-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 6ec96c2b8dccfd02b257c734dcf0524d columnFamilyName a 2024-11-24T08:29:56,211 DEBUG [StoreOpener-6ec96c2b8dccfd02b257c734dcf0524d-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:29:56,217 DEBUG [StoreOpener-6ec96c2b8dccfd02b257c734dcf0524d-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:46165/hbase/data/default/testReplayEditsWrittenViaHRegion/6ec96c2b8dccfd02b257c734dcf0524d/a/c19e76f417474e619eb423745b198822 2024-11-24T08:29:56,217 INFO [StoreOpener-6ec96c2b8dccfd02b257c734dcf0524d-1 {}] regionserver.HStore(327): Store=6ec96c2b8dccfd02b257c734dcf0524d/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T08:29:56,217 INFO [StoreOpener-6ec96c2b8dccfd02b257c734dcf0524d-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 6ec96c2b8dccfd02b257c734dcf0524d 2024-11-24T08:29:56,219 INFO [StoreOpener-6ec96c2b8dccfd02b257c734dcf0524d-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 6ec96c2b8dccfd02b257c734dcf0524d columnFamilyName b 2024-11-24T08:29:56,219 DEBUG [StoreOpener-6ec96c2b8dccfd02b257c734dcf0524d-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:29:56,219 INFO [StoreOpener-6ec96c2b8dccfd02b257c734dcf0524d-1 {}] regionserver.HStore(327): Store=6ec96c2b8dccfd02b257c734dcf0524d/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T08:29:56,219 INFO [StoreOpener-6ec96c2b8dccfd02b257c734dcf0524d-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 6ec96c2b8dccfd02b257c734dcf0524d 2024-11-24T08:29:56,220 INFO [StoreOpener-6ec96c2b8dccfd02b257c734dcf0524d-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 6ec96c2b8dccfd02b257c734dcf0524d columnFamilyName c 2024-11-24T08:29:56,220 DEBUG [StoreOpener-6ec96c2b8dccfd02b257c734dcf0524d-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:29:56,227 DEBUG [StoreOpener-6ec96c2b8dccfd02b257c734dcf0524d-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:46165/hbase/data/default/testReplayEditsWrittenViaHRegion/6ec96c2b8dccfd02b257c734dcf0524d/c/9c44fbe8921d40908e624e046d28f719 2024-11-24T08:29:56,227 INFO [StoreOpener-6ec96c2b8dccfd02b257c734dcf0524d-1 {}] regionserver.HStore(327): Store=6ec96c2b8dccfd02b257c734dcf0524d/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T08:29:56,227 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 6ec96c2b8dccfd02b257c734dcf0524d 2024-11-24T08:29:56,228 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46165/hbase/data/default/testReplayEditsWrittenViaHRegion/6ec96c2b8dccfd02b257c734dcf0524d 2024-11-24T08:29:56,230 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:46165/hbase/data/default/testReplayEditsWrittenViaHRegion/6ec96c2b8dccfd02b257c734dcf0524d 2024-11-24T08:29:56,231 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:46165/hbase/data/default/testReplayEditsWrittenViaHRegion/6ec96c2b8dccfd02b257c734dcf0524d/recovered.edits/0000000000000000032 2024-11-24T08:29:56,240 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:46165/hbase/data/default/testReplayEditsWrittenViaHRegion/6ec96c2b8dccfd02b257c734dcf0524d/recovered.edits/0000000000000000032: isRecoveredEdits=true, hasTagCompression=true, hasValueCompression=false, valueCompressionType=null 2024-11-24T08:29:56,242 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 10, skipped 20, firstSequenceIdInLog=3, maxSequenceIdInLog=32, path=hdfs://localhost:46165/hbase/data/default/testReplayEditsWrittenViaHRegion/6ec96c2b8dccfd02b257c734dcf0524d/recovered.edits/0000000000000000032 2024-11-24T08:29:56,242 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 6ec96c2b8dccfd02b257c734dcf0524d 3/3 column families, dataSize=870 B heapSize=2.31 KB 2024-11-24T08:29:56,275 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46165/hbase/data/default/testReplayEditsWrittenViaHRegion/6ec96c2b8dccfd02b257c734dcf0524d/.tmp/b/ab516b2aa228480282c68c3402fef655 is 91, key is testReplayEditsWrittenViaHRegion/b:x0/1732436995928/Put/seqid=0 2024-11-24T08:29:56,293 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33555 is added to blk_1073741900_1078 (size=5958) 2024-11-24T08:29:56,294 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34459 is added to blk_1073741900_1078 (size=5958) 2024-11-24T08:29:56,294 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=32 (bloomFilter=true), to=hdfs://localhost:46165/hbase/data/default/testReplayEditsWrittenViaHRegion/6ec96c2b8dccfd02b257c734dcf0524d/.tmp/b/ab516b2aa228480282c68c3402fef655 2024-11-24T08:29:56,296 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40533 is added to blk_1073741900_1078 (size=5958) 2024-11-24T08:29:56,302 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46165/hbase/data/default/testReplayEditsWrittenViaHRegion/6ec96c2b8dccfd02b257c734dcf0524d/.tmp/b/ab516b2aa228480282c68c3402fef655 as hdfs://localhost:46165/hbase/data/default/testReplayEditsWrittenViaHRegion/6ec96c2b8dccfd02b257c734dcf0524d/b/ab516b2aa228480282c68c3402fef655 2024-11-24T08:29:56,310 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46165/hbase/data/default/testReplayEditsWrittenViaHRegion/6ec96c2b8dccfd02b257c734dcf0524d/b/ab516b2aa228480282c68c3402fef655, entries=10, sequenceid=32, filesize=5.8 K 2024-11-24T08:29:56,311 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~870 B/870, heapSize ~1.80 KB/1840, currentSize=0 B/0 for 6ec96c2b8dccfd02b257c734dcf0524d in 69ms, sequenceid=32, compaction requested=false; wal=null 2024-11-24T08:29:56,312 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:46165/hbase/data/default/testReplayEditsWrittenViaHRegion/6ec96c2b8dccfd02b257c734dcf0524d/recovered.edits/0000000000000000032 2024-11-24T08:29:56,313 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 6ec96c2b8dccfd02b257c734dcf0524d 2024-11-24T08:29:56,313 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 6ec96c2b8dccfd02b257c734dcf0524d 2024-11-24T08:29:56,314 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsWrittenViaHRegion descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-24T08:29:56,318 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 6ec96c2b8dccfd02b257c734dcf0524d 2024-11-24T08:29:56,322 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:46165/hbase/data/default/testReplayEditsWrittenViaHRegion/6ec96c2b8dccfd02b257c734dcf0524d/recovered.edits/33.seqid, newMaxSeqId=33, maxSeqId=1 2024-11-24T08:29:56,323 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 6ec96c2b8dccfd02b257c734dcf0524d; next sequenceid=34; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=72514329, jitterRate=0.08054770529270172}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-24T08:29:56,324 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 6ec96c2b8dccfd02b257c734dcf0524d: Writing region info on filesystem at 1732436996208Initializing all the Stores at 1732436996209 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732436996209Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732436996209Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732436996209Obtaining lock to block concurrent updates at 1732436996243 (+34 ms)Preparing flush snapshotting stores in 6ec96c2b8dccfd02b257c734dcf0524d at 1732436996243Finished memstore snapshotting testReplayEditsWrittenViaHRegion,,1732436995784.6ec96c2b8dccfd02b257c734dcf0524d., syncing WAL and waiting on mvcc, flushsize=dataSize=870, getHeapSize=2320, getOffHeapSize=0, getCellsCount=10 at 1732436996243Flushing stores of testReplayEditsWrittenViaHRegion,,1732436995784.6ec96c2b8dccfd02b257c734dcf0524d. at 1732436996243Flushing 6ec96c2b8dccfd02b257c734dcf0524d/b: creating writer at 1732436996243Flushing 6ec96c2b8dccfd02b257c734dcf0524d/b: appending metadata at 1732436996274 (+31 ms)Flushing 6ec96c2b8dccfd02b257c734dcf0524d/b: closing flushed file at 1732436996274Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@15715247: reopening flushed file at 1732436996301 (+27 ms)Finished flush of dataSize ~870 B/870, heapSize ~1.80 KB/1840, currentSize=0 B/0 for 6ec96c2b8dccfd02b257c734dcf0524d in 69ms, sequenceid=32, compaction requested=false; wal=null at 1732436996311 (+10 ms)Cleaning up temporary data from old regions at 1732436996313 (+2 ms)Region opened successfully at 1732436996324 (+11 ms) 2024-11-24T08:29:56,351 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestAsyncWALReplayCompressed#testReplayEditsAfterPartialFlush Thread=417 (was 408) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1541013681_22 at /127.0.0.1:58834 [Waiting for operation #30] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-20-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1541013681_22 at /127.0.0.1:58984 [Receiving block BP-1907186785-172.17.0.2-1732436973683:blk_1073741899_1077] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-20-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-20-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1541013681_22 at /127.0.0.1:53818 [Receiving block BP-1907186785-172.17.0.2-1732436973683:blk_1073741899_1077] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1541013681_22 at /127.0.0.1:47700 [Receiving block BP-1907186785-172.17.0.2-1732436973683:blk_1073741899_1077] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-1907186785-172.17.0.2-1732436973683:blk_1073741899_1077, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-1907186785-172.17.0.2-1732436973683:blk_1073741899_1077, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-1907186785-172.17.0.2-1732436973683:blk_1073741899_1077, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1541013681_22 at /127.0.0.1:47590 [Waiting for operation #42] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1541013681_22 at /127.0.0.1:53688 [Waiting for operation #30] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=1127 (was 1051) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=169 (was 169), ProcessCount=11 (was 11), AvailableMemoryMB=6656 (was 6673) 2024-11-24T08:29:56,351 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1127 is superior to 1024 2024-11-24T08:29:56,364 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestAsyncWALReplayCompressed#testReplayEditsAfterAbortingFlush Thread=417, OpenFileDescriptor=1127, MaxFileDescriptor=1048576, SystemLoadAverage=169, ProcessCount=11, AvailableMemoryMB=6652 2024-11-24T08:29:56,364 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1127 is superior to 1024 2024-11-24T08:29:56,381 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-24T08:29:56,384 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-24T08:29:56,384 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-24T08:29:56,388 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-98456540, suffix=, logDir=hdfs://localhost:46165/hbase/WALs/hregion-98456540, archiveDir=hdfs://localhost:46165/hbase/oldWALs, maxLogs=32 2024-11-24T08:29:56,403 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-98456540/hregion-98456540.1732436996388, exclude list is [], retry=0 2024-11-24T08:29:56,407 DEBUG [AsyncFSWAL-22-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40533,DS-5eeb9666-6dd5-4a48-9fb1-3f9aff1acdca,DISK] 2024-11-24T08:29:56,409 DEBUG [AsyncFSWAL-22-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33555,DS-8d97c3e2-1525-418c-a970-00a7157a0d31,DISK] 2024-11-24T08:29:56,413 DEBUG [AsyncFSWAL-22-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:34459,DS-6b4d1656-f49b-4a16-be49-2eb682c16e11,DISK] 2024-11-24T08:29:56,418 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-98456540/hregion-98456540.1732436996388 2024-11-24T08:29:56,419 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:42673:42673),(127.0.0.1/127.0.0.1:39357:39357),(127.0.0.1/127.0.0.1:44131:44131)] 2024-11-24T08:29:56,419 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => dd50c4d5b440a4bb2a9b27e4f8a18fc9, NAME => 'testReplayEditsAfterAbortingFlush,,1732436996382.dd50c4d5b440a4bb2a9b27e4f8a18fc9.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testReplayEditsAfterAbortingFlush', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:46165/hbase 2024-11-24T08:29:56,428 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40533 is added to blk_1073741902_1080 (size=68) 2024-11-24T08:29:56,428 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33555 is added to blk_1073741902_1080 (size=68) 2024-11-24T08:29:56,428 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34459 is added to blk_1073741902_1080 (size=68) 2024-11-24T08:29:56,429 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsAfterAbortingFlush,,1732436996382.dd50c4d5b440a4bb2a9b27e4f8a18fc9.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T08:29:56,432 INFO [StoreOpener-dd50c4d5b440a4bb2a9b27e4f8a18fc9-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region dd50c4d5b440a4bb2a9b27e4f8a18fc9 2024-11-24T08:29:56,434 INFO [StoreOpener-dd50c4d5b440a4bb2a9b27e4f8a18fc9-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region dd50c4d5b440a4bb2a9b27e4f8a18fc9 columnFamilyName a 2024-11-24T08:29:56,434 DEBUG [StoreOpener-dd50c4d5b440a4bb2a9b27e4f8a18fc9-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:29:56,434 INFO [StoreOpener-dd50c4d5b440a4bb2a9b27e4f8a18fc9-1 {}] regionserver.HStore(327): Store=dd50c4d5b440a4bb2a9b27e4f8a18fc9/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T08:29:56,435 INFO [StoreOpener-dd50c4d5b440a4bb2a9b27e4f8a18fc9-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region dd50c4d5b440a4bb2a9b27e4f8a18fc9 2024-11-24T08:29:56,436 INFO [StoreOpener-dd50c4d5b440a4bb2a9b27e4f8a18fc9-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region dd50c4d5b440a4bb2a9b27e4f8a18fc9 columnFamilyName b 2024-11-24T08:29:56,436 DEBUG [StoreOpener-dd50c4d5b440a4bb2a9b27e4f8a18fc9-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:29:56,436 INFO [StoreOpener-dd50c4d5b440a4bb2a9b27e4f8a18fc9-1 {}] regionserver.HStore(327): Store=dd50c4d5b440a4bb2a9b27e4f8a18fc9/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T08:29:56,436 INFO [StoreOpener-dd50c4d5b440a4bb2a9b27e4f8a18fc9-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region dd50c4d5b440a4bb2a9b27e4f8a18fc9 2024-11-24T08:29:56,438 INFO [StoreOpener-dd50c4d5b440a4bb2a9b27e4f8a18fc9-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region dd50c4d5b440a4bb2a9b27e4f8a18fc9 columnFamilyName c 2024-11-24T08:29:56,438 DEBUG [StoreOpener-dd50c4d5b440a4bb2a9b27e4f8a18fc9-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:29:56,438 INFO [StoreOpener-dd50c4d5b440a4bb2a9b27e4f8a18fc9-1 {}] regionserver.HStore(327): Store=dd50c4d5b440a4bb2a9b27e4f8a18fc9/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T08:29:56,438 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for dd50c4d5b440a4bb2a9b27e4f8a18fc9 2024-11-24T08:29:56,439 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46165/hbase/data/default/testReplayEditsAfterAbortingFlush/dd50c4d5b440a4bb2a9b27e4f8a18fc9 2024-11-24T08:29:56,440 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46165/hbase/data/default/testReplayEditsAfterAbortingFlush/dd50c4d5b440a4bb2a9b27e4f8a18fc9 2024-11-24T08:29:56,441 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for dd50c4d5b440a4bb2a9b27e4f8a18fc9 2024-11-24T08:29:56,441 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for dd50c4d5b440a4bb2a9b27e4f8a18fc9 2024-11-24T08:29:56,441 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsAfterAbortingFlush descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-24T08:29:56,443 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for dd50c4d5b440a4bb2a9b27e4f8a18fc9 2024-11-24T08:29:56,449 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:46165/hbase/data/default/testReplayEditsAfterAbortingFlush/dd50c4d5b440a4bb2a9b27e4f8a18fc9/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-24T08:29:56,449 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened dd50c4d5b440a4bb2a9b27e4f8a18fc9; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=70766981, jitterRate=0.05451019108295441}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-24T08:29:56,450 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for dd50c4d5b440a4bb2a9b27e4f8a18fc9: Writing region info on filesystem at 1732436996429Initializing all the Stores at 1732436996430 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732436996430Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732436996432 (+2 ms)Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732436996432Cleaning up temporary data from old regions at 1732436996441 (+9 ms)Region opened successfully at 1732436996450 (+9 ms) 2024-11-24T08:29:56,450 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing dd50c4d5b440a4bb2a9b27e4f8a18fc9, disabling compactions & flushes 2024-11-24T08:29:56,450 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsAfterAbortingFlush,,1732436996382.dd50c4d5b440a4bb2a9b27e4f8a18fc9. 2024-11-24T08:29:56,450 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsAfterAbortingFlush,,1732436996382.dd50c4d5b440a4bb2a9b27e4f8a18fc9. 2024-11-24T08:29:56,450 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsAfterAbortingFlush,,1732436996382.dd50c4d5b440a4bb2a9b27e4f8a18fc9. after waiting 0 ms 2024-11-24T08:29:56,450 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsAfterAbortingFlush,,1732436996382.dd50c4d5b440a4bb2a9b27e4f8a18fc9. 2024-11-24T08:29:56,455 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsAfterAbortingFlush,,1732436996382.dd50c4d5b440a4bb2a9b27e4f8a18fc9. 2024-11-24T08:29:56,455 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for dd50c4d5b440a4bb2a9b27e4f8a18fc9: Waiting for close lock at 1732436996450Disabling compacts and flushes for region at 1732436996450Disabling writes for close at 1732436996450Writing region close event to WAL at 1732436996455 (+5 ms)Closed at 1732436996455 2024-11-24T08:29:56,459 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34459 is added to blk_1073741901_1079 (size=93) 2024-11-24T08:29:56,459 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40533 is added to blk_1073741901_1079 (size=93) 2024-11-24T08:29:56,459 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33555 is added to blk_1073741901_1079 (size=93) 2024-11-24T08:29:56,461 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-11-24T08:29:56,461 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-98456540:(num 1732436996388) 2024-11-24T08:29:56,461 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-24T08:29:56,463 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:46165/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1732436996380, archiveDir=hdfs://localhost:46165/hbase/oldWALs, maxLogs=32 2024-11-24T08:29:56,481 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1732436996380/wal.1732436996464, exclude list is [], retry=0 2024-11-24T08:29:56,483 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40533,DS-5eeb9666-6dd5-4a48-9fb1-3f9aff1acdca,DISK] 2024-11-24T08:29:56,484 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:34459,DS-6b4d1656-f49b-4a16-be49-2eb682c16e11,DISK] 2024-11-24T08:29:56,484 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33555,DS-8d97c3e2-1525-418c-a970-00a7157a0d31,DISK] 2024-11-24T08:29:56,486 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1732436996380/wal.1732436996464 2024-11-24T08:29:56,486 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:42673:42673),(127.0.0.1/127.0.0.1:44131:44131),(127.0.0.1/127.0.0.1:39357:39357)] 2024-11-24T08:29:56,546 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => dd50c4d5b440a4bb2a9b27e4f8a18fc9, NAME => 'testReplayEditsAfterAbortingFlush,,1732436996382.dd50c4d5b440a4bb2a9b27e4f8a18fc9.', STARTKEY => '', ENDKEY => ''} 2024-11-24T08:29:56,549 DEBUG [Time-limited test {}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testReplayEditsAfterAbortingFlush dd50c4d5b440a4bb2a9b27e4f8a18fc9 2024-11-24T08:29:56,549 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsAfterAbortingFlush,,1732436996382.dd50c4d5b440a4bb2a9b27e4f8a18fc9.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T08:29:56,549 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for dd50c4d5b440a4bb2a9b27e4f8a18fc9 2024-11-24T08:29:56,549 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for dd50c4d5b440a4bb2a9b27e4f8a18fc9 2024-11-24T08:29:56,551 INFO [StoreOpener-dd50c4d5b440a4bb2a9b27e4f8a18fc9-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region dd50c4d5b440a4bb2a9b27e4f8a18fc9 2024-11-24T08:29:56,552 INFO [StoreOpener-dd50c4d5b440a4bb2a9b27e4f8a18fc9-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region dd50c4d5b440a4bb2a9b27e4f8a18fc9 columnFamilyName a 2024-11-24T08:29:56,552 DEBUG [StoreOpener-dd50c4d5b440a4bb2a9b27e4f8a18fc9-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:29:56,553 INFO [StoreOpener-dd50c4d5b440a4bb2a9b27e4f8a18fc9-1 {}] regionserver.HStore(327): Store=dd50c4d5b440a4bb2a9b27e4f8a18fc9/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T08:29:56,553 INFO [StoreOpener-dd50c4d5b440a4bb2a9b27e4f8a18fc9-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region dd50c4d5b440a4bb2a9b27e4f8a18fc9 2024-11-24T08:29:56,554 INFO [StoreOpener-dd50c4d5b440a4bb2a9b27e4f8a18fc9-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region dd50c4d5b440a4bb2a9b27e4f8a18fc9 columnFamilyName b 2024-11-24T08:29:56,554 DEBUG [StoreOpener-dd50c4d5b440a4bb2a9b27e4f8a18fc9-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:29:56,555 INFO [StoreOpener-dd50c4d5b440a4bb2a9b27e4f8a18fc9-1 {}] regionserver.HStore(327): Store=dd50c4d5b440a4bb2a9b27e4f8a18fc9/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T08:29:56,555 INFO [StoreOpener-dd50c4d5b440a4bb2a9b27e4f8a18fc9-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region dd50c4d5b440a4bb2a9b27e4f8a18fc9 2024-11-24T08:29:56,556 INFO [StoreOpener-dd50c4d5b440a4bb2a9b27e4f8a18fc9-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region dd50c4d5b440a4bb2a9b27e4f8a18fc9 columnFamilyName c 2024-11-24T08:29:56,556 DEBUG [StoreOpener-dd50c4d5b440a4bb2a9b27e4f8a18fc9-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:29:56,557 INFO [StoreOpener-dd50c4d5b440a4bb2a9b27e4f8a18fc9-1 {}] regionserver.HStore(327): Store=dd50c4d5b440a4bb2a9b27e4f8a18fc9/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T08:29:56,557 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for dd50c4d5b440a4bb2a9b27e4f8a18fc9 2024-11-24T08:29:56,558 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46165/hbase/data/default/testReplayEditsAfterAbortingFlush/dd50c4d5b440a4bb2a9b27e4f8a18fc9 2024-11-24T08:29:56,559 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46165/hbase/data/default/testReplayEditsAfterAbortingFlush/dd50c4d5b440a4bb2a9b27e4f8a18fc9 2024-11-24T08:29:56,560 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for dd50c4d5b440a4bb2a9b27e4f8a18fc9 2024-11-24T08:29:56,560 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for dd50c4d5b440a4bb2a9b27e4f8a18fc9 2024-11-24T08:29:56,561 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsAfterAbortingFlush descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-24T08:29:56,562 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for dd50c4d5b440a4bb2a9b27e4f8a18fc9 2024-11-24T08:29:56,563 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened dd50c4d5b440a4bb2a9b27e4f8a18fc9; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=68427807, jitterRate=0.01965378224849701}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-24T08:29:56,563 DEBUG [Time-limited test {}] regionserver.HRegion(1122): Running coprocessor post-open hooks for dd50c4d5b440a4bb2a9b27e4f8a18fc9 2024-11-24T08:29:56,563 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for dd50c4d5b440a4bb2a9b27e4f8a18fc9: Running coprocessor pre-open hook at 1732436996549Writing region info on filesystem at 1732436996549Initializing all the Stores at 1732436996550 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732436996551 (+1 ms)Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732436996551Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732436996551Cleaning up temporary data from old regions at 1732436996560 (+9 ms)Running coprocessor post-open hooks at 1732436996563 (+3 ms)Region opened successfully at 1732436996563 2024-11-24T08:29:56,580 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing dd50c4d5b440a4bb2a9b27e4f8a18fc9 3/3 column families, dataSize=590 B heapSize=2.08 KB 2024-11-24T08:29:56,581 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for dd50c4d5b440a4bb2a9b27e4f8a18fc9/a, retrying num=0 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T08:29:57,257 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testReplayEditsWrittenViaHRegion 2024-11-24T08:29:57,257 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testReplayEditsWrittenViaHRegion Metrics about Tables on a single HBase RegionServer 2024-11-24T08:29:57,259 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testSequentialEditLogSeqNum 2024-11-24T08:29:57,259 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testSequentialEditLogSeqNum Metrics about Tables on a single HBase RegionServer 2024-11-24T08:29:57,260 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testReplayEditsAfterRegionMovedWithMultiCF 2024-11-24T08:29:57,260 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testReplayEditsAfterRegionMovedWithMultiCF Metrics about Tables on a single HBase RegionServer 2024-11-24T08:29:57,261 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testRegionMadeOfBulkLoadedFilesOnly 2024-11-24T08:29:57,261 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testRegionMadeOfBulkLoadedFilesOnly Metrics about Tables on a single HBase RegionServer 2024-11-24T08:29:57,262 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testReplayEditsAfterAbortingFlush 2024-11-24T08:29:57,262 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testReplayEditsAfterAbortingFlush Metrics about Tables on a single HBase RegionServer 2024-11-24T08:29:57,581 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for dd50c4d5b440a4bb2a9b27e4f8a18fc9/a, retrying num=1 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T08:29:58,252 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-24T08:29:58,582 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for dd50c4d5b440a4bb2a9b27e4f8a18fc9/a, retrying num=2 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T08:29:59,583 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for dd50c4d5b440a4bb2a9b27e4f8a18fc9/a, retrying num=3 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T08:30:00,583 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for dd50c4d5b440a4bb2a9b27e4f8a18fc9/a, retrying num=4 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T08:30:01,584 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for dd50c4d5b440a4bb2a9b27e4f8a18fc9/a, retrying num=5 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T08:30:02,585 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for dd50c4d5b440a4bb2a9b27e4f8a18fc9/a, retrying num=6 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T08:30:03,586 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for dd50c4d5b440a4bb2a9b27e4f8a18fc9/a, retrying num=7 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T08:30:04,587 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for dd50c4d5b440a4bb2a9b27e4f8a18fc9/a, retrying num=8 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T08:30:05,587 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for dd50c4d5b440a4bb2a9b27e4f8a18fc9/a, retrying num=9 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T08:30:05,588 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for dd50c4d5b440a4bb2a9b27e4f8a18fc9: 2024-11-24T08:30:05,588 INFO [Time-limited test {}] wal.AbstractTestWALReplay(671): Expected simulated exception when flushing region, region: testReplayEditsAfterAbortingFlush,,1732436996382.dd50c4d5b440a4bb2a9b27e4f8a18fc9. 2024-11-24T08:30:05,603 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for dd50c4d5b440a4bb2a9b27e4f8a18fc9: 2024-11-24T08:30:05,603 INFO [Time-limited test {}] wal.AbstractTestWALReplay(691): Expected exception when flushing region because server is stopped,Aborting flush because server is aborted... 2024-11-24T08:30:05,604 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing dd50c4d5b440a4bb2a9b27e4f8a18fc9, disabling compactions & flushes 2024-11-24T08:30:05,604 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsAfterAbortingFlush,,1732436996382.dd50c4d5b440a4bb2a9b27e4f8a18fc9. 2024-11-24T08:30:05,604 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsAfterAbortingFlush,,1732436996382.dd50c4d5b440a4bb2a9b27e4f8a18fc9. 2024-11-24T08:30:05,604 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsAfterAbortingFlush,,1732436996382.dd50c4d5b440a4bb2a9b27e4f8a18fc9. after waiting 0 ms 2024-11-24T08:30:05,604 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsAfterAbortingFlush,,1732436996382.dd50c4d5b440a4bb2a9b27e4f8a18fc9. 2024-11-24T08:30:05,604 ERROR [Time-limited test {}] regionserver.HRegion(1960): Memstore data size is 1190 in region testReplayEditsAfterAbortingFlush,,1732436996382.dd50c4d5b440a4bb2a9b27e4f8a18fc9. 2024-11-24T08:30:05,604 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsAfterAbortingFlush,,1732436996382.dd50c4d5b440a4bb2a9b27e4f8a18fc9. 2024-11-24T08:30:05,604 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for dd50c4d5b440a4bb2a9b27e4f8a18fc9: Waiting for close lock at 1732437005604Running coprocessor pre-close hooks at 1732437005604Disabling compacts and flushes for region at 1732437005604Disabling writes for close at 1732437005604Writing region close event to WAL at 1732437005604Running coprocessor post-close hooks at 1732437005604Closed at 1732437005604 2024-11-24T08:30:05,607 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40533 is added to blk_1073741903_1081 (size=2398) 2024-11-24T08:30:05,607 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34459 is added to blk_1073741903_1081 (size=2398) 2024-11-24T08:30:05,607 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33555 is added to blk_1073741903_1081 (size=2398) 2024-11-24T08:30:05,623 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:46165/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1732436996380/wal.1732436996464, size=2.3 K (2398bytes) 2024-11-24T08:30:05,623 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:46165/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1732436996380/wal.1732436996464 2024-11-24T08:30:05,624 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:46165/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1732436996380/wal.1732436996464 after 1ms 2024-11-24T08:30:05,627 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:46165/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1732436996380/wal.1732436996464: isRecoveredEdits=false, hasTagCompression=true, hasValueCompression=false, valueCompressionType=null 2024-11-24T08:30:05,627 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:46165/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1732436996380/wal.1732436996464 took 4ms 2024-11-24T08:30:05,629 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:46165/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1732436996380/wal.1732436996464 so closing down 2024-11-24T08:30:05,629 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-24T08:30:05,630 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000004-wal.1732436996464.temp 2024-11-24T08:30:05,632 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:46165/hbase/data/default/testReplayEditsAfterAbortingFlush/dd50c4d5b440a4bb2a9b27e4f8a18fc9/recovered.edits/0000000000000000004-wal.1732436996464.temp 2024-11-24T08:30:05,632 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-11-24T08:30:05,639 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33555 is added to blk_1073741904_1082 (size=1672) 2024-11-24T08:30:05,640 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34459 is added to blk_1073741904_1082 (size=1672) 2024-11-24T08:30:05,640 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40533 is added to blk_1073741904_1082 (size=1672) 2024-11-24T08:30:05,640 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:46165/hbase/data/default/testReplayEditsAfterAbortingFlush/dd50c4d5b440a4bb2a9b27e4f8a18fc9/recovered.edits/0000000000000000004-wal.1732436996464.temp (wrote 20 edits, skipped 0 edits in 0 ms) 2024-11-24T08:30:05,642 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:46165/hbase/data/default/testReplayEditsAfterAbortingFlush/dd50c4d5b440a4bb2a9b27e4f8a18fc9/recovered.edits/0000000000000000004-wal.1732436996464.temp to hdfs://localhost:46165/hbase/data/default/testReplayEditsAfterAbortingFlush/dd50c4d5b440a4bb2a9b27e4f8a18fc9/recovered.edits/0000000000000000026 2024-11-24T08:30:05,642 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 23 edits across 1 Regions in 15 ms; skipped=3; WAL=hdfs://localhost:46165/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1732436996380/wal.1732436996464, size=2.3 K, length=2398, corrupted=false, cancelled=false 2024-11-24T08:30:05,642 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:46165/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1732436996380/wal.1732436996464, journal: Splitting hdfs://localhost:46165/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1732436996380/wal.1732436996464, size=2.3 K (2398bytes) at 1732437005623Finishing writing output for hdfs://localhost:46165/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1732436996380/wal.1732436996464 so closing down at 1732437005629 (+6 ms)Creating recovered edits writer path=hdfs://localhost:46165/hbase/data/default/testReplayEditsAfterAbortingFlush/dd50c4d5b440a4bb2a9b27e4f8a18fc9/recovered.edits/0000000000000000004-wal.1732436996464.temp at 1732437005632 (+3 ms)3 split writer threads finished at 1732437005632Closed recovered edits writer path=hdfs://localhost:46165/hbase/data/default/testReplayEditsAfterAbortingFlush/dd50c4d5b440a4bb2a9b27e4f8a18fc9/recovered.edits/0000000000000000004-wal.1732436996464.temp (wrote 20 edits, skipped 0 edits in 0 ms) at 1732437005640 (+8 ms)Rename recovered edits hdfs://localhost:46165/hbase/data/default/testReplayEditsAfterAbortingFlush/dd50c4d5b440a4bb2a9b27e4f8a18fc9/recovered.edits/0000000000000000004-wal.1732436996464.temp to hdfs://localhost:46165/hbase/data/default/testReplayEditsAfterAbortingFlush/dd50c4d5b440a4bb2a9b27e4f8a18fc9/recovered.edits/0000000000000000026 at 1732437005642 (+2 ms)Processed 23 edits across 1 Regions in 15 ms; skipped=3; WAL=hdfs://localhost:46165/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1732436996380/wal.1732436996464, size=2.3 K, length=2398, corrupted=false, cancelled=false at 1732437005642 2024-11-24T08:30:05,644 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:46165/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1732436996380/wal.1732436996464 to hdfs://localhost:46165/hbase/oldWALs/wal.1732436996464 2024-11-24T08:30:05,645 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:46165/hbase/data/default/testReplayEditsAfterAbortingFlush/dd50c4d5b440a4bb2a9b27e4f8a18fc9/recovered.edits/0000000000000000026 2024-11-24T08:30:05,645 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-24T08:30:05,646 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:46165/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1732436996380, archiveDir=hdfs://localhost:46165/hbase/oldWALs, maxLogs=32 2024-11-24T08:30:05,660 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1732436996380/wal.1732437005647, exclude list is [], retry=0 2024-11-24T08:30:05,663 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40533,DS-5eeb9666-6dd5-4a48-9fb1-3f9aff1acdca,DISK] 2024-11-24T08:30:05,663 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33555,DS-8d97c3e2-1525-418c-a970-00a7157a0d31,DISK] 2024-11-24T08:30:05,663 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:34459,DS-6b4d1656-f49b-4a16-be49-2eb682c16e11,DISK] 2024-11-24T08:30:05,665 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1732436996380/wal.1732437005647 2024-11-24T08:30:05,665 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:42673:42673),(127.0.0.1/127.0.0.1:39357:39357),(127.0.0.1/127.0.0.1:44131:44131)] 2024-11-24T08:30:05,666 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => dd50c4d5b440a4bb2a9b27e4f8a18fc9, NAME => 'testReplayEditsAfterAbortingFlush,,1732436996382.dd50c4d5b440a4bb2a9b27e4f8a18fc9.', STARTKEY => '', ENDKEY => ''} 2024-11-24T08:30:05,666 DEBUG [Time-limited test {}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testReplayEditsAfterAbortingFlush dd50c4d5b440a4bb2a9b27e4f8a18fc9 2024-11-24T08:30:05,666 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsAfterAbortingFlush,,1732436996382.dd50c4d5b440a4bb2a9b27e4f8a18fc9.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T08:30:05,666 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for dd50c4d5b440a4bb2a9b27e4f8a18fc9 2024-11-24T08:30:05,666 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for dd50c4d5b440a4bb2a9b27e4f8a18fc9 2024-11-24T08:30:05,668 INFO [StoreOpener-dd50c4d5b440a4bb2a9b27e4f8a18fc9-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region dd50c4d5b440a4bb2a9b27e4f8a18fc9 2024-11-24T08:30:05,669 INFO [StoreOpener-dd50c4d5b440a4bb2a9b27e4f8a18fc9-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region dd50c4d5b440a4bb2a9b27e4f8a18fc9 columnFamilyName a 2024-11-24T08:30:05,669 DEBUG [StoreOpener-dd50c4d5b440a4bb2a9b27e4f8a18fc9-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:30:05,670 INFO [StoreOpener-dd50c4d5b440a4bb2a9b27e4f8a18fc9-1 {}] regionserver.HStore(327): Store=dd50c4d5b440a4bb2a9b27e4f8a18fc9/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T08:30:05,670 INFO [StoreOpener-dd50c4d5b440a4bb2a9b27e4f8a18fc9-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region dd50c4d5b440a4bb2a9b27e4f8a18fc9 2024-11-24T08:30:05,671 INFO [StoreOpener-dd50c4d5b440a4bb2a9b27e4f8a18fc9-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region dd50c4d5b440a4bb2a9b27e4f8a18fc9 columnFamilyName b 2024-11-24T08:30:05,671 DEBUG [StoreOpener-dd50c4d5b440a4bb2a9b27e4f8a18fc9-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:30:05,671 INFO [StoreOpener-dd50c4d5b440a4bb2a9b27e4f8a18fc9-1 {}] regionserver.HStore(327): Store=dd50c4d5b440a4bb2a9b27e4f8a18fc9/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T08:30:05,671 INFO [StoreOpener-dd50c4d5b440a4bb2a9b27e4f8a18fc9-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region dd50c4d5b440a4bb2a9b27e4f8a18fc9 2024-11-24T08:30:05,672 INFO [StoreOpener-dd50c4d5b440a4bb2a9b27e4f8a18fc9-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region dd50c4d5b440a4bb2a9b27e4f8a18fc9 columnFamilyName c 2024-11-24T08:30:05,672 DEBUG [StoreOpener-dd50c4d5b440a4bb2a9b27e4f8a18fc9-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:30:05,673 INFO [StoreOpener-dd50c4d5b440a4bb2a9b27e4f8a18fc9-1 {}] regionserver.HStore(327): Store=dd50c4d5b440a4bb2a9b27e4f8a18fc9/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T08:30:05,673 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for dd50c4d5b440a4bb2a9b27e4f8a18fc9 2024-11-24T08:30:05,673 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46165/hbase/data/default/testReplayEditsAfterAbortingFlush/dd50c4d5b440a4bb2a9b27e4f8a18fc9 2024-11-24T08:30:05,675 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:46165/hbase/data/default/testReplayEditsAfterAbortingFlush/dd50c4d5b440a4bb2a9b27e4f8a18fc9 2024-11-24T08:30:05,675 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:46165/hbase/data/default/testReplayEditsAfterAbortingFlush/dd50c4d5b440a4bb2a9b27e4f8a18fc9/recovered.edits/0000000000000000026 2024-11-24T08:30:05,677 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:46165/hbase/data/default/testReplayEditsAfterAbortingFlush/dd50c4d5b440a4bb2a9b27e4f8a18fc9/recovered.edits/0000000000000000026: isRecoveredEdits=true, hasTagCompression=true, hasValueCompression=false, valueCompressionType=null 2024-11-24T08:30:05,679 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 20, skipped 0, firstSequenceIdInLog=4, maxSequenceIdInLog=26, path=hdfs://localhost:46165/hbase/data/default/testReplayEditsAfterAbortingFlush/dd50c4d5b440a4bb2a9b27e4f8a18fc9/recovered.edits/0000000000000000026 2024-11-24T08:30:05,679 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing dd50c4d5b440a4bb2a9b27e4f8a18fc9 3/3 column families, dataSize=1.16 KB heapSize=3.41 KB 2024-11-24T08:30:05,695 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46165/hbase/data/default/testReplayEditsAfterAbortingFlush/dd50c4d5b440a4bb2a9b27e4f8a18fc9/.tmp/a/ff661033e3f74745a78bb77619bb3bae is 64, key is testReplayEditsAfterAbortingFlush12/a:q/1732437005594/Put/seqid=0 2024-11-24T08:30:05,702 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33555 is added to blk_1073741906_1084 (size=5523) 2024-11-24T08:30:05,702 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40533 is added to blk_1073741906_1084 (size=5523) 2024-11-24T08:30:05,703 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34459 is added to blk_1073741906_1084 (size=5523) 2024-11-24T08:30:05,704 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=416 B at sequenceid=26 (bloomFilter=true), to=hdfs://localhost:46165/hbase/data/default/testReplayEditsAfterAbortingFlush/dd50c4d5b440a4bb2a9b27e4f8a18fc9/.tmp/a/ff661033e3f74745a78bb77619bb3bae 2024-11-24T08:30:05,725 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46165/hbase/data/default/testReplayEditsAfterAbortingFlush/dd50c4d5b440a4bb2a9b27e4f8a18fc9/.tmp/b/8613d7b9a9a442cc9cd445ac384c6f63 is 64, key is testReplayEditsAfterAbortingFlush10/b:q/1732437005589/Put/seqid=0 2024-11-24T08:30:05,734 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34459 is added to blk_1073741907_1085 (size=5524) 2024-11-24T08:30:05,736 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40533 is added to blk_1073741907_1085 (size=5524) 2024-11-24T08:30:05,736 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33555 is added to blk_1073741907_1085 (size=5524) 2024-11-24T08:30:05,737 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=417 B at sequenceid=26 (bloomFilter=true), to=hdfs://localhost:46165/hbase/data/default/testReplayEditsAfterAbortingFlush/dd50c4d5b440a4bb2a9b27e4f8a18fc9/.tmp/b/8613d7b9a9a442cc9cd445ac384c6f63 2024-11-24T08:30:05,765 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46165/hbase/data/default/testReplayEditsAfterAbortingFlush/dd50c4d5b440a4bb2a9b27e4f8a18fc9/.tmp/c/52a88bd63fc048b8887efd0b99350b23 is 64, key is testReplayEditsAfterAbortingFlush11/c:q/1732437005591/Put/seqid=0 2024-11-24T08:30:05,771 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33555 is added to blk_1073741908_1086 (size=5457) 2024-11-24T08:30:05,771 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34459 is added to blk_1073741908_1086 (size=5457) 2024-11-24T08:30:05,772 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40533 is added to blk_1073741908_1086 (size=5457) 2024-11-24T08:30:05,773 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=357 B at sequenceid=26 (bloomFilter=true), to=hdfs://localhost:46165/hbase/data/default/testReplayEditsAfterAbortingFlush/dd50c4d5b440a4bb2a9b27e4f8a18fc9/.tmp/c/52a88bd63fc048b8887efd0b99350b23 2024-11-24T08:30:05,779 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46165/hbase/data/default/testReplayEditsAfterAbortingFlush/dd50c4d5b440a4bb2a9b27e4f8a18fc9/.tmp/a/ff661033e3f74745a78bb77619bb3bae as hdfs://localhost:46165/hbase/data/default/testReplayEditsAfterAbortingFlush/dd50c4d5b440a4bb2a9b27e4f8a18fc9/a/ff661033e3f74745a78bb77619bb3bae 2024-11-24T08:30:05,786 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46165/hbase/data/default/testReplayEditsAfterAbortingFlush/dd50c4d5b440a4bb2a9b27e4f8a18fc9/a/ff661033e3f74745a78bb77619bb3bae, entries=7, sequenceid=26, filesize=5.4 K 2024-11-24T08:30:05,787 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46165/hbase/data/default/testReplayEditsAfterAbortingFlush/dd50c4d5b440a4bb2a9b27e4f8a18fc9/.tmp/b/8613d7b9a9a442cc9cd445ac384c6f63 as hdfs://localhost:46165/hbase/data/default/testReplayEditsAfterAbortingFlush/dd50c4d5b440a4bb2a9b27e4f8a18fc9/b/8613d7b9a9a442cc9cd445ac384c6f63 2024-11-24T08:30:05,793 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46165/hbase/data/default/testReplayEditsAfterAbortingFlush/dd50c4d5b440a4bb2a9b27e4f8a18fc9/b/8613d7b9a9a442cc9cd445ac384c6f63, entries=7, sequenceid=26, filesize=5.4 K 2024-11-24T08:30:05,794 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46165/hbase/data/default/testReplayEditsAfterAbortingFlush/dd50c4d5b440a4bb2a9b27e4f8a18fc9/.tmp/c/52a88bd63fc048b8887efd0b99350b23 as hdfs://localhost:46165/hbase/data/default/testReplayEditsAfterAbortingFlush/dd50c4d5b440a4bb2a9b27e4f8a18fc9/c/52a88bd63fc048b8887efd0b99350b23 2024-11-24T08:30:05,800 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46165/hbase/data/default/testReplayEditsAfterAbortingFlush/dd50c4d5b440a4bb2a9b27e4f8a18fc9/c/52a88bd63fc048b8887efd0b99350b23, entries=6, sequenceid=26, filesize=5.3 K 2024-11-24T08:30:05,800 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~1.16 KB/1190, heapSize ~3.36 KB/3440, currentSize=0 B/0 for dd50c4d5b440a4bb2a9b27e4f8a18fc9 in 121ms, sequenceid=26, compaction requested=false; wal=null 2024-11-24T08:30:05,801 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:46165/hbase/data/default/testReplayEditsAfterAbortingFlush/dd50c4d5b440a4bb2a9b27e4f8a18fc9/recovered.edits/0000000000000000026 2024-11-24T08:30:05,802 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for dd50c4d5b440a4bb2a9b27e4f8a18fc9 2024-11-24T08:30:05,802 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for dd50c4d5b440a4bb2a9b27e4f8a18fc9 2024-11-24T08:30:05,803 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsAfterAbortingFlush descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-24T08:30:05,804 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for dd50c4d5b440a4bb2a9b27e4f8a18fc9 2024-11-24T08:30:05,806 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:46165/hbase/data/default/testReplayEditsAfterAbortingFlush/dd50c4d5b440a4bb2a9b27e4f8a18fc9/recovered.edits/26.seqid, newMaxSeqId=26, maxSeqId=1 2024-11-24T08:30:05,807 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened dd50c4d5b440a4bb2a9b27e4f8a18fc9; next sequenceid=27; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=65421347, jitterRate=-0.025145962834358215}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-24T08:30:05,807 DEBUG [Time-limited test {}] regionserver.HRegion(1122): Running coprocessor post-open hooks for dd50c4d5b440a4bb2a9b27e4f8a18fc9 2024-11-24T08:30:05,808 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for dd50c4d5b440a4bb2a9b27e4f8a18fc9: Running coprocessor pre-open hook at 1732437005666Writing region info on filesystem at 1732437005666Initializing all the Stores at 1732437005667 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732437005667Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732437005667Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732437005668 (+1 ms)Obtaining lock to block concurrent updates at 1732437005679 (+11 ms)Preparing flush snapshotting stores in dd50c4d5b440a4bb2a9b27e4f8a18fc9 at 1732437005679Finished memstore snapshotting testReplayEditsAfterAbortingFlush,,1732436996382.dd50c4d5b440a4bb2a9b27e4f8a18fc9., syncing WAL and waiting on mvcc, flushsize=dataSize=1190, getHeapSize=3440, getOffHeapSize=0, getCellsCount=20 at 1732437005679Flushing stores of testReplayEditsAfterAbortingFlush,,1732436996382.dd50c4d5b440a4bb2a9b27e4f8a18fc9. at 1732437005679Flushing dd50c4d5b440a4bb2a9b27e4f8a18fc9/a: creating writer at 1732437005679Flushing dd50c4d5b440a4bb2a9b27e4f8a18fc9/a: appending metadata at 1732437005694 (+15 ms)Flushing dd50c4d5b440a4bb2a9b27e4f8a18fc9/a: closing flushed file at 1732437005694Flushing dd50c4d5b440a4bb2a9b27e4f8a18fc9/b: creating writer at 1732437005710 (+16 ms)Flushing dd50c4d5b440a4bb2a9b27e4f8a18fc9/b: appending metadata at 1732437005724 (+14 ms)Flushing dd50c4d5b440a4bb2a9b27e4f8a18fc9/b: closing flushed file at 1732437005724Flushing dd50c4d5b440a4bb2a9b27e4f8a18fc9/c: creating writer at 1732437005742 (+18 ms)Flushing dd50c4d5b440a4bb2a9b27e4f8a18fc9/c: appending metadata at 1732437005764 (+22 ms)Flushing dd50c4d5b440a4bb2a9b27e4f8a18fc9/c: closing flushed file at 1732437005764Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@4da995f1: reopening flushed file at 1732437005778 (+14 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@43f108db: reopening flushed file at 1732437005786 (+8 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@4fa3166e: reopening flushed file at 1732437005793 (+7 ms)Finished flush of dataSize ~1.16 KB/1190, heapSize ~3.36 KB/3440, currentSize=0 B/0 for dd50c4d5b440a4bb2a9b27e4f8a18fc9 in 121ms, sequenceid=26, compaction requested=false; wal=null at 1732437005800 (+7 ms)Cleaning up temporary data from old regions at 1732437005802 (+2 ms)Running coprocessor post-open hooks at 1732437005807 (+5 ms)Region opened successfully at 1732437005808 (+1 ms) 2024-11-24T08:30:05,834 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestAsyncWALReplayCompressed#testReplayEditsAfterAbortingFlush Thread=413 (was 417), OpenFileDescriptor=1179 (was 1127) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=210 (was 169) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=6192 (was 6652) 2024-11-24T08:30:05,834 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1179 is superior to 1024 2024-11-24T08:30:05,848 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestAsyncWALReplayCompressed#testDatalossWhenInputError Thread=413, OpenFileDescriptor=1179, MaxFileDescriptor=1048576, SystemLoadAverage=210, ProcessCount=11, AvailableMemoryMB=6191 2024-11-24T08:30:05,848 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1179 is superior to 1024 2024-11-24T08:30:05,863 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-24T08:30:05,865 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-24T08:30:05,866 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-24T08:30:05,868 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-30411996, suffix=, logDir=hdfs://localhost:46165/hbase/WALs/hregion-30411996, archiveDir=hdfs://localhost:46165/hbase/oldWALs, maxLogs=32 2024-11-24T08:30:05,882 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-30411996/hregion-30411996.1732437005868, exclude list is [], retry=0 2024-11-24T08:30:05,885 DEBUG [AsyncFSWAL-24-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33555,DS-8d97c3e2-1525-418c-a970-00a7157a0d31,DISK] 2024-11-24T08:30:05,885 DEBUG [AsyncFSWAL-24-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:34459,DS-6b4d1656-f49b-4a16-be49-2eb682c16e11,DISK] 2024-11-24T08:30:05,886 DEBUG [AsyncFSWAL-24-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40533,DS-5eeb9666-6dd5-4a48-9fb1-3f9aff1acdca,DISK] 2024-11-24T08:30:05,888 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-30411996/hregion-30411996.1732437005868 2024-11-24T08:30:05,892 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:39357:39357),(127.0.0.1/127.0.0.1:42673:42673),(127.0.0.1/127.0.0.1:44131:44131)] 2024-11-24T08:30:05,892 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => 5e30eecc0239a4e6ecc3ea03333f9da0, NAME => 'testDatalossWhenInputError,,1732437005864.5e30eecc0239a4e6ecc3ea03333f9da0.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testDatalossWhenInputError', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:46165/hbase 2024-11-24T08:30:05,904 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34459 is added to blk_1073741910_1088 (size=61) 2024-11-24T08:30:05,904 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40533 is added to blk_1073741910_1088 (size=61) 2024-11-24T08:30:05,905 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33555 is added to blk_1073741910_1088 (size=61) 2024-11-24T08:30:05,905 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testDatalossWhenInputError,,1732437005864.5e30eecc0239a4e6ecc3ea03333f9da0.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T08:30:05,906 INFO [StoreOpener-5e30eecc0239a4e6ecc3ea03333f9da0-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 5e30eecc0239a4e6ecc3ea03333f9da0 2024-11-24T08:30:05,908 INFO [StoreOpener-5e30eecc0239a4e6ecc3ea03333f9da0-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 5e30eecc0239a4e6ecc3ea03333f9da0 columnFamilyName a 2024-11-24T08:30:05,908 DEBUG [StoreOpener-5e30eecc0239a4e6ecc3ea03333f9da0-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:30:05,908 INFO [StoreOpener-5e30eecc0239a4e6ecc3ea03333f9da0-1 {}] regionserver.HStore(327): Store=5e30eecc0239a4e6ecc3ea03333f9da0/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T08:30:05,908 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 5e30eecc0239a4e6ecc3ea03333f9da0 2024-11-24T08:30:05,909 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46165/hbase/data/default/testDatalossWhenInputError/5e30eecc0239a4e6ecc3ea03333f9da0 2024-11-24T08:30:05,909 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46165/hbase/data/default/testDatalossWhenInputError/5e30eecc0239a4e6ecc3ea03333f9da0 2024-11-24T08:30:05,910 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 5e30eecc0239a4e6ecc3ea03333f9da0 2024-11-24T08:30:05,910 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 5e30eecc0239a4e6ecc3ea03333f9da0 2024-11-24T08:30:05,912 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 5e30eecc0239a4e6ecc3ea03333f9da0 2024-11-24T08:30:05,914 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:46165/hbase/data/default/testDatalossWhenInputError/5e30eecc0239a4e6ecc3ea03333f9da0/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-24T08:30:05,914 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 5e30eecc0239a4e6ecc3ea03333f9da0; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=74512090, jitterRate=0.11031666398048401}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-24T08:30:05,915 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 5e30eecc0239a4e6ecc3ea03333f9da0: Writing region info on filesystem at 1732437005905Initializing all the Stores at 1732437005906 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732437005906Cleaning up temporary data from old regions at 1732437005910 (+4 ms)Region opened successfully at 1732437005915 (+5 ms) 2024-11-24T08:30:05,916 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 5e30eecc0239a4e6ecc3ea03333f9da0, disabling compactions & flushes 2024-11-24T08:30:05,916 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testDatalossWhenInputError,,1732437005864.5e30eecc0239a4e6ecc3ea03333f9da0. 2024-11-24T08:30:05,916 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testDatalossWhenInputError,,1732437005864.5e30eecc0239a4e6ecc3ea03333f9da0. 2024-11-24T08:30:05,916 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testDatalossWhenInputError,,1732437005864.5e30eecc0239a4e6ecc3ea03333f9da0. after waiting 0 ms 2024-11-24T08:30:05,916 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testDatalossWhenInputError,,1732437005864.5e30eecc0239a4e6ecc3ea03333f9da0. 2024-11-24T08:30:05,916 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testDatalossWhenInputError,,1732437005864.5e30eecc0239a4e6ecc3ea03333f9da0. 2024-11-24T08:30:05,916 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 5e30eecc0239a4e6ecc3ea03333f9da0: Waiting for close lock at 1732437005916Disabling compacts and flushes for region at 1732437005916Disabling writes for close at 1732437005916Writing region close event to WAL at 1732437005916Closed at 1732437005916 2024-11-24T08:30:05,919 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33555 is added to blk_1073741909_1087 (size=93) 2024-11-24T08:30:05,919 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34459 is added to blk_1073741909_1087 (size=93) 2024-11-24T08:30:05,919 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40533 is added to blk_1073741909_1087 (size=93) 2024-11-24T08:30:05,923 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-11-24T08:30:05,923 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-30411996:(num 1732437005868) 2024-11-24T08:30:05,923 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-24T08:30:05,925 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:46165/hbase/WALs/testdatalosswheninputerror-manual,16010,1732437005862, archiveDir=hdfs://localhost:46165/hbase/oldWALs, maxLogs=32 2024-11-24T08:30:05,939 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testdatalosswheninputerror-manual,16010,1732437005862/wal.1732437005925, exclude list is [], retry=0 2024-11-24T08:30:05,942 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33555,DS-8d97c3e2-1525-418c-a970-00a7157a0d31,DISK] 2024-11-24T08:30:05,942 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40533,DS-5eeb9666-6dd5-4a48-9fb1-3f9aff1acdca,DISK] 2024-11-24T08:30:05,942 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:34459,DS-6b4d1656-f49b-4a16-be49-2eb682c16e11,DISK] 2024-11-24T08:30:05,944 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testdatalosswheninputerror-manual,16010,1732437005862/wal.1732437005925 2024-11-24T08:30:05,944 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:39357:39357),(127.0.0.1/127.0.0.1:42673:42673),(127.0.0.1/127.0.0.1:44131:44131)] 2024-11-24T08:30:05,944 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 5e30eecc0239a4e6ecc3ea03333f9da0, NAME => 'testDatalossWhenInputError,,1732437005864.5e30eecc0239a4e6ecc3ea03333f9da0.', STARTKEY => '', ENDKEY => ''} 2024-11-24T08:30:05,945 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testDatalossWhenInputError,,1732437005864.5e30eecc0239a4e6ecc3ea03333f9da0.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T08:30:05,945 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 5e30eecc0239a4e6ecc3ea03333f9da0 2024-11-24T08:30:05,945 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 5e30eecc0239a4e6ecc3ea03333f9da0 2024-11-24T08:30:05,946 INFO [StoreOpener-5e30eecc0239a4e6ecc3ea03333f9da0-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 5e30eecc0239a4e6ecc3ea03333f9da0 2024-11-24T08:30:05,947 INFO [StoreOpener-5e30eecc0239a4e6ecc3ea03333f9da0-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 5e30eecc0239a4e6ecc3ea03333f9da0 columnFamilyName a 2024-11-24T08:30:05,947 DEBUG [StoreOpener-5e30eecc0239a4e6ecc3ea03333f9da0-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:30:05,948 INFO [StoreOpener-5e30eecc0239a4e6ecc3ea03333f9da0-1 {}] regionserver.HStore(327): Store=5e30eecc0239a4e6ecc3ea03333f9da0/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T08:30:05,948 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 5e30eecc0239a4e6ecc3ea03333f9da0 2024-11-24T08:30:05,948 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46165/hbase/data/default/testDatalossWhenInputError/5e30eecc0239a4e6ecc3ea03333f9da0 2024-11-24T08:30:05,949 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46165/hbase/data/default/testDatalossWhenInputError/5e30eecc0239a4e6ecc3ea03333f9da0 2024-11-24T08:30:05,950 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 5e30eecc0239a4e6ecc3ea03333f9da0 2024-11-24T08:30:05,950 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 5e30eecc0239a4e6ecc3ea03333f9da0 2024-11-24T08:30:05,951 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 5e30eecc0239a4e6ecc3ea03333f9da0 2024-11-24T08:30:05,952 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 5e30eecc0239a4e6ecc3ea03333f9da0; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=61912447, jitterRate=-0.07743264734745026}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-24T08:30:05,953 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 5e30eecc0239a4e6ecc3ea03333f9da0: Writing region info on filesystem at 1732437005945Initializing all the Stores at 1732437005946 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732437005946Cleaning up temporary data from old regions at 1732437005950 (+4 ms)Region opened successfully at 1732437005953 (+3 ms) 2024-11-24T08:30:05,957 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-24T08:30:05,961 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 5e30eecc0239a4e6ecc3ea03333f9da0, disabling compactions & flushes 2024-11-24T08:30:05,961 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testDatalossWhenInputError,,1732437005864.5e30eecc0239a4e6ecc3ea03333f9da0. 2024-11-24T08:30:05,961 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testDatalossWhenInputError,,1732437005864.5e30eecc0239a4e6ecc3ea03333f9da0. 2024-11-24T08:30:05,961 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testDatalossWhenInputError,,1732437005864.5e30eecc0239a4e6ecc3ea03333f9da0. after waiting 0 ms 2024-11-24T08:30:05,961 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testDatalossWhenInputError,,1732437005864.5e30eecc0239a4e6ecc3ea03333f9da0. 2024-11-24T08:30:05,961 ERROR [Time-limited test {}] regionserver.HRegion(1960): Memstore data size is 750 in region testDatalossWhenInputError,,1732437005864.5e30eecc0239a4e6ecc3ea03333f9da0. 2024-11-24T08:30:05,962 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testDatalossWhenInputError,,1732437005864.5e30eecc0239a4e6ecc3ea03333f9da0. 2024-11-24T08:30:05,962 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 5e30eecc0239a4e6ecc3ea03333f9da0: Waiting for close lock at 1732437005961Disabling compacts and flushes for region at 1732437005961Disabling writes for close at 1732437005961Writing region close event to WAL at 1732437005961Closed at 1732437005961 2024-11-24T08:30:05,964 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40533 is added to blk_1073741911_1089 (size=838) 2024-11-24T08:30:05,964 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33555 is added to blk_1073741911_1089 (size=838) 2024-11-24T08:30:05,964 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34459 is added to blk_1073741911_1089 (size=838) 2024-11-24T08:30:05,985 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:46165/hbase/WALs/testdatalosswheninputerror-manual,16010,1732437005862/wal.1732437005925, size=838 (838bytes) 2024-11-24T08:30:05,986 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:46165/hbase/WALs/testdatalosswheninputerror-manual,16010,1732437005862/wal.1732437005925 2024-11-24T08:30:05,986 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:46165/hbase/WALs/testdatalosswheninputerror-manual,16010,1732437005862/wal.1732437005925 after 0ms 2024-11-24T08:30:05,988 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:46165/hbase/WALs/testdatalosswheninputerror-manual,16010,1732437005862/wal.1732437005925: isRecoveredEdits=false, hasTagCompression=true, hasValueCompression=false, valueCompressionType=null 2024-11-24T08:30:05,989 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:46165/hbase/WALs/testdatalosswheninputerror-manual,16010,1732437005862/wal.1732437005925 took 4ms 2024-11-24T08:30:05,993 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:46165/hbase/WALs/testdatalosswheninputerror-manual,16010,1732437005862/wal.1732437005925 so closing down 2024-11-24T08:30:05,993 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-24T08:30:05,994 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000003-wal.1732437005925.temp 2024-11-24T08:30:05,996 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:46165/hbase/data/default/testDatalossWhenInputError/5e30eecc0239a4e6ecc3ea03333f9da0/recovered.edits/0000000000000000003-wal.1732437005925.temp 2024-11-24T08:30:06,001 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-11-24T08:30:06,012 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34459 is added to blk_1073741912_1090 (size=838) 2024-11-24T08:30:06,012 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33555 is added to blk_1073741912_1090 (size=838) 2024-11-24T08:30:06,013 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40533 is added to blk_1073741912_1090 (size=838) 2024-11-24T08:30:06,013 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:46165/hbase/data/default/testDatalossWhenInputError/5e30eecc0239a4e6ecc3ea03333f9da0/recovered.edits/0000000000000000003-wal.1732437005925.temp (wrote 10 edits, skipped 0 edits in 0 ms) 2024-11-24T08:30:06,015 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:46165/hbase/data/default/testDatalossWhenInputError/5e30eecc0239a4e6ecc3ea03333f9da0/recovered.edits/0000000000000000003-wal.1732437005925.temp to hdfs://localhost:46165/hbase/data/default/testDatalossWhenInputError/5e30eecc0239a4e6ecc3ea03333f9da0/recovered.edits/0000000000000000012 2024-11-24T08:30:06,015 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 10 edits across 1 Regions in 24 ms; skipped=0; WAL=hdfs://localhost:46165/hbase/WALs/testdatalosswheninputerror-manual,16010,1732437005862/wal.1732437005925, size=838, length=838, corrupted=false, cancelled=false 2024-11-24T08:30:06,015 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:46165/hbase/WALs/testdatalosswheninputerror-manual,16010,1732437005862/wal.1732437005925, journal: Splitting hdfs://localhost:46165/hbase/WALs/testdatalosswheninputerror-manual,16010,1732437005862/wal.1732437005925, size=838 (838bytes) at 1732437005986Finishing writing output for hdfs://localhost:46165/hbase/WALs/testdatalosswheninputerror-manual,16010,1732437005862/wal.1732437005925 so closing down at 1732437005993 (+7 ms)Creating recovered edits writer path=hdfs://localhost:46165/hbase/data/default/testDatalossWhenInputError/5e30eecc0239a4e6ecc3ea03333f9da0/recovered.edits/0000000000000000003-wal.1732437005925.temp at 1732437005996 (+3 ms)3 split writer threads finished at 1732437006001 (+5 ms)Closed recovered edits writer path=hdfs://localhost:46165/hbase/data/default/testDatalossWhenInputError/5e30eecc0239a4e6ecc3ea03333f9da0/recovered.edits/0000000000000000003-wal.1732437005925.temp (wrote 10 edits, skipped 0 edits in 0 ms) at 1732437006013 (+12 ms)Rename recovered edits hdfs://localhost:46165/hbase/data/default/testDatalossWhenInputError/5e30eecc0239a4e6ecc3ea03333f9da0/recovered.edits/0000000000000000003-wal.1732437005925.temp to hdfs://localhost:46165/hbase/data/default/testDatalossWhenInputError/5e30eecc0239a4e6ecc3ea03333f9da0/recovered.edits/0000000000000000012 at 1732437006015 (+2 ms)Processed 10 edits across 1 Regions in 24 ms; skipped=0; WAL=hdfs://localhost:46165/hbase/WALs/testdatalosswheninputerror-manual,16010,1732437005862/wal.1732437005925, size=838, length=838, corrupted=false, cancelled=false at 1732437006015 2024-11-24T08:30:06,017 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:46165/hbase/WALs/testdatalosswheninputerror-manual,16010,1732437005862/wal.1732437005925 to hdfs://localhost:46165/hbase/oldWALs/wal.1732437005925 2024-11-24T08:30:06,018 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:46165/hbase/data/default/testDatalossWhenInputError/5e30eecc0239a4e6ecc3ea03333f9da0/recovered.edits/0000000000000000012 2024-11-24T08:30:06,022 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:46165/hbase/data/default/testDatalossWhenInputError/5e30eecc0239a4e6ecc3ea03333f9da0/recovered.edits/0000000000000000012: isRecoveredEdits=true, hasTagCompression=true, hasValueCompression=false, valueCompressionType=null 2024-11-24T08:30:06,367 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-24T08:30:06,370 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:46165/hbase/WALs/testdatalosswheninputerror-manual,16010,1732437005862, archiveDir=hdfs://localhost:46165/hbase/oldWALs, maxLogs=32 2024-11-24T08:30:06,383 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testdatalosswheninputerror-manual,16010,1732437005862/wal.1732437006370, exclude list is [], retry=0 2024-11-24T08:30:06,386 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40533,DS-5eeb9666-6dd5-4a48-9fb1-3f9aff1acdca,DISK] 2024-11-24T08:30:06,386 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33555,DS-8d97c3e2-1525-418c-a970-00a7157a0d31,DISK] 2024-11-24T08:30:06,387 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:34459,DS-6b4d1656-f49b-4a16-be49-2eb682c16e11,DISK] 2024-11-24T08:30:06,391 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testdatalosswheninputerror-manual,16010,1732437005862/wal.1732437006370 2024-11-24T08:30:06,391 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:42673:42673),(127.0.0.1/127.0.0.1:39357:39357),(127.0.0.1/127.0.0.1:44131:44131)] 2024-11-24T08:30:06,391 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 5e30eecc0239a4e6ecc3ea03333f9da0, NAME => 'testDatalossWhenInputError,,1732437005864.5e30eecc0239a4e6ecc3ea03333f9da0.', STARTKEY => '', ENDKEY => ''} 2024-11-24T08:30:06,391 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testDatalossWhenInputError,,1732437005864.5e30eecc0239a4e6ecc3ea03333f9da0.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T08:30:06,391 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 5e30eecc0239a4e6ecc3ea03333f9da0 2024-11-24T08:30:06,391 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 5e30eecc0239a4e6ecc3ea03333f9da0 2024-11-24T08:30:06,394 INFO [StoreOpener-5e30eecc0239a4e6ecc3ea03333f9da0-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 5e30eecc0239a4e6ecc3ea03333f9da0 2024-11-24T08:30:06,396 INFO [StoreOpener-5e30eecc0239a4e6ecc3ea03333f9da0-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 5e30eecc0239a4e6ecc3ea03333f9da0 columnFamilyName a 2024-11-24T08:30:06,396 DEBUG [StoreOpener-5e30eecc0239a4e6ecc3ea03333f9da0-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:30:06,397 INFO [StoreOpener-5e30eecc0239a4e6ecc3ea03333f9da0-1 {}] regionserver.HStore(327): Store=5e30eecc0239a4e6ecc3ea03333f9da0/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T08:30:06,397 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 5e30eecc0239a4e6ecc3ea03333f9da0 2024-11-24T08:30:06,398 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46165/hbase/data/default/testDatalossWhenInputError/5e30eecc0239a4e6ecc3ea03333f9da0 2024-11-24T08:30:06,399 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:46165/hbase/data/default/testDatalossWhenInputError/5e30eecc0239a4e6ecc3ea03333f9da0 2024-11-24T08:30:06,400 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:46165/hbase/data/default/testDatalossWhenInputError/5e30eecc0239a4e6ecc3ea03333f9da0/recovered.edits/0000000000000000012 2024-11-24T08:30:06,402 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:46165/hbase/data/default/testDatalossWhenInputError/5e30eecc0239a4e6ecc3ea03333f9da0/recovered.edits/0000000000000000012: isRecoveredEdits=true, hasTagCompression=true, hasValueCompression=false, valueCompressionType=null 2024-11-24T08:30:06,403 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 10, skipped 0, firstSequenceIdInLog=3, maxSequenceIdInLog=12, path=hdfs://localhost:46165/hbase/data/default/testDatalossWhenInputError/5e30eecc0239a4e6ecc3ea03333f9da0/recovered.edits/0000000000000000012 2024-11-24T08:30:06,403 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 5e30eecc0239a4e6ecc3ea03333f9da0 1/1 column families, dataSize=750 B heapSize=1.73 KB 2024-11-24T08:30:06,423 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46165/hbase/data/default/testDatalossWhenInputError/5e30eecc0239a4e6ecc3ea03333f9da0/.tmp/a/5055fe6fb351484686cd4fe5b02022a5 is 79, key is testDatalossWhenInputError/a:x0/1732437005953/Put/seqid=0 2024-11-24T08:30:06,430 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33555 is added to blk_1073741914_1092 (size=5808) 2024-11-24T08:30:06,430 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40533 is added to blk_1073741914_1092 (size=5808) 2024-11-24T08:30:06,431 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34459 is added to blk_1073741914_1092 (size=5808) 2024-11-24T08:30:06,431 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=750 B at sequenceid=12 (bloomFilter=true), to=hdfs://localhost:46165/hbase/data/default/testDatalossWhenInputError/5e30eecc0239a4e6ecc3ea03333f9da0/.tmp/a/5055fe6fb351484686cd4fe5b02022a5 2024-11-24T08:30:06,440 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46165/hbase/data/default/testDatalossWhenInputError/5e30eecc0239a4e6ecc3ea03333f9da0/.tmp/a/5055fe6fb351484686cd4fe5b02022a5 as hdfs://localhost:46165/hbase/data/default/testDatalossWhenInputError/5e30eecc0239a4e6ecc3ea03333f9da0/a/5055fe6fb351484686cd4fe5b02022a5 2024-11-24T08:30:06,450 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46165/hbase/data/default/testDatalossWhenInputError/5e30eecc0239a4e6ecc3ea03333f9da0/a/5055fe6fb351484686cd4fe5b02022a5, entries=10, sequenceid=12, filesize=5.7 K 2024-11-24T08:30:06,451 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~750 B/750, heapSize ~1.72 KB/1760, currentSize=0 B/0 for 5e30eecc0239a4e6ecc3ea03333f9da0 in 47ms, sequenceid=12, compaction requested=false; wal=null 2024-11-24T08:30:06,451 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:46165/hbase/data/default/testDatalossWhenInputError/5e30eecc0239a4e6ecc3ea03333f9da0/recovered.edits/0000000000000000012 2024-11-24T08:30:06,452 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 5e30eecc0239a4e6ecc3ea03333f9da0 2024-11-24T08:30:06,452 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 5e30eecc0239a4e6ecc3ea03333f9da0 2024-11-24T08:30:06,456 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 5e30eecc0239a4e6ecc3ea03333f9da0 2024-11-24T08:30:06,459 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:46165/hbase/data/default/testDatalossWhenInputError/5e30eecc0239a4e6ecc3ea03333f9da0/recovered.edits/12.seqid, newMaxSeqId=12, maxSeqId=1 2024-11-24T08:30:06,460 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 5e30eecc0239a4e6ecc3ea03333f9da0; next sequenceid=13; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=64649485, jitterRate=-0.036647602915763855}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-24T08:30:06,461 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 5e30eecc0239a4e6ecc3ea03333f9da0: Writing region info on filesystem at 1732437006392Initializing all the Stores at 1732437006394 (+2 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732437006394Obtaining lock to block concurrent updates at 1732437006403 (+9 ms)Preparing flush snapshotting stores in 5e30eecc0239a4e6ecc3ea03333f9da0 at 1732437006403Finished memstore snapshotting testDatalossWhenInputError,,1732437005864.5e30eecc0239a4e6ecc3ea03333f9da0., syncing WAL and waiting on mvcc, flushsize=dataSize=750, getHeapSize=1760, getOffHeapSize=0, getCellsCount=10 at 1732437006403Flushing stores of testDatalossWhenInputError,,1732437005864.5e30eecc0239a4e6ecc3ea03333f9da0. at 1732437006403Flushing 5e30eecc0239a4e6ecc3ea03333f9da0/a: creating writer at 1732437006403Flushing 5e30eecc0239a4e6ecc3ea03333f9da0/a: appending metadata at 1732437006422 (+19 ms)Flushing 5e30eecc0239a4e6ecc3ea03333f9da0/a: closing flushed file at 1732437006422Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@532f0b64: reopening flushed file at 1732437006439 (+17 ms)Finished flush of dataSize ~750 B/750, heapSize ~1.72 KB/1760, currentSize=0 B/0 for 5e30eecc0239a4e6ecc3ea03333f9da0 in 47ms, sequenceid=12, compaction requested=false; wal=null at 1732437006451 (+12 ms)Cleaning up temporary data from old regions at 1732437006452 (+1 ms)Region opened successfully at 1732437006461 (+9 ms) 2024-11-24T08:30:06,464 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 5e30eecc0239a4e6ecc3ea03333f9da0, NAME => 'testDatalossWhenInputError,,1732437005864.5e30eecc0239a4e6ecc3ea03333f9da0.', STARTKEY => '', ENDKEY => ''} 2024-11-24T08:30:06,465 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testDatalossWhenInputError,,1732437005864.5e30eecc0239a4e6ecc3ea03333f9da0.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T08:30:06,465 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 5e30eecc0239a4e6ecc3ea03333f9da0 2024-11-24T08:30:06,465 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 5e30eecc0239a4e6ecc3ea03333f9da0 2024-11-24T08:30:06,466 INFO [StoreOpener-5e30eecc0239a4e6ecc3ea03333f9da0-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 5e30eecc0239a4e6ecc3ea03333f9da0 2024-11-24T08:30:06,467 INFO [StoreOpener-5e30eecc0239a4e6ecc3ea03333f9da0-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 5e30eecc0239a4e6ecc3ea03333f9da0 columnFamilyName a 2024-11-24T08:30:06,467 DEBUG [StoreOpener-5e30eecc0239a4e6ecc3ea03333f9da0-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:30:06,474 DEBUG [StoreOpener-5e30eecc0239a4e6ecc3ea03333f9da0-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:46165/hbase/data/default/testDatalossWhenInputError/5e30eecc0239a4e6ecc3ea03333f9da0/a/5055fe6fb351484686cd4fe5b02022a5 2024-11-24T08:30:06,474 INFO [StoreOpener-5e30eecc0239a4e6ecc3ea03333f9da0-1 {}] regionserver.HStore(327): Store=5e30eecc0239a4e6ecc3ea03333f9da0/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T08:30:06,474 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 5e30eecc0239a4e6ecc3ea03333f9da0 2024-11-24T08:30:06,475 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46165/hbase/data/default/testDatalossWhenInputError/5e30eecc0239a4e6ecc3ea03333f9da0 2024-11-24T08:30:06,476 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46165/hbase/data/default/testDatalossWhenInputError/5e30eecc0239a4e6ecc3ea03333f9da0 2024-11-24T08:30:06,476 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 5e30eecc0239a4e6ecc3ea03333f9da0 2024-11-24T08:30:06,476 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 5e30eecc0239a4e6ecc3ea03333f9da0 2024-11-24T08:30:06,478 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 5e30eecc0239a4e6ecc3ea03333f9da0 2024-11-24T08:30:06,480 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:46165/hbase/data/default/testDatalossWhenInputError/5e30eecc0239a4e6ecc3ea03333f9da0/recovered.edits/13.seqid, newMaxSeqId=13, maxSeqId=12 2024-11-24T08:30:06,482 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 5e30eecc0239a4e6ecc3ea03333f9da0; next sequenceid=14; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=75255561, jitterRate=0.12139524519443512}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-24T08:30:06,482 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 5e30eecc0239a4e6ecc3ea03333f9da0: Writing region info on filesystem at 1732437006465Initializing all the Stores at 1732437006466 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732437006466Cleaning up temporary data from old regions at 1732437006476 (+10 ms)Region opened successfully at 1732437006482 (+6 ms) 2024-11-24T08:30:06,501 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestAsyncWALReplayCompressed#testDatalossWhenInputError Thread=425 (was 413) Potentially hanging thread: PacketResponder: BP-1907186785-172.17.0.2-1732436973683:blk_1073741913_1091, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-24-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1541013681_22 at /127.0.0.1:50644 [Receiving block BP-1907186785-172.17.0.2-1732436973683:blk_1073741913_1091] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1541013681_22 at /127.0.0.1:35748 [Waiting for operation #28] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1541013681_22 at /127.0.0.1:42804 [Receiving block BP-1907186785-172.17.0.2-1732436973683:blk_1073741913_1091] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1541013681_22 at /127.0.0.1:50658 [Waiting for operation #11] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-1907186785-172.17.0.2-1732436973683:blk_1073741913_1091, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-24-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-1907186785-172.17.0.2-1732436973683:blk_1073741913_1091, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1541013681_22 at /127.0.0.1:35846 [Receiving block BP-1907186785-172.17.0.2-1732436973683:blk_1073741913_1091] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-24-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1541013681_22 at /127.0.0.1:42818 [Waiting for operation #3] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=1262 (was 1179) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=210 (was 210), ProcessCount=11 (was 11), AvailableMemoryMB=6174 (was 6191) 2024-11-24T08:30:06,502 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1262 is superior to 1024 2024-11-24T08:30:06,514 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestAsyncWALReplayCompressed#testCompactedBulkLoadedFiles Thread=425, OpenFileDescriptor=1262, MaxFileDescriptor=1048576, SystemLoadAverage=210, ProcessCount=11, AvailableMemoryMB=6172 2024-11-24T08:30:06,515 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1262 is superior to 1024 2024-11-24T08:30:06,530 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-24T08:30:06,531 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-24T08:30:06,532 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-24T08:30:06,535 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-28817925, suffix=, logDir=hdfs://localhost:46165/hbase/WALs/hregion-28817925, archiveDir=hdfs://localhost:46165/hbase/oldWALs, maxLogs=32 2024-11-24T08:30:06,549 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-28817925/hregion-28817925.1732437006535, exclude list is [], retry=0 2024-11-24T08:30:06,552 DEBUG [AsyncFSWAL-26-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40533,DS-5eeb9666-6dd5-4a48-9fb1-3f9aff1acdca,DISK] 2024-11-24T08:30:06,553 DEBUG [AsyncFSWAL-26-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:34459,DS-6b4d1656-f49b-4a16-be49-2eb682c16e11,DISK] 2024-11-24T08:30:06,554 DEBUG [AsyncFSWAL-26-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33555,DS-8d97c3e2-1525-418c-a970-00a7157a0d31,DISK] 2024-11-24T08:30:06,561 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-28817925/hregion-28817925.1732437006535 2024-11-24T08:30:06,561 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:42673:42673),(127.0.0.1/127.0.0.1:44131:44131),(127.0.0.1/127.0.0.1:39357:39357)] 2024-11-24T08:30:06,561 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => d8f62349b1f39c8b7fcb4535307a9738, NAME => 'testCompactedBulkLoadedFiles,,1732437006530.d8f62349b1f39c8b7fcb4535307a9738.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testCompactedBulkLoadedFiles', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:46165/hbase 2024-11-24T08:30:06,575 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40533 is added to blk_1073741916_1094 (size=63) 2024-11-24T08:30:06,575 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34459 is added to blk_1073741916_1094 (size=63) 2024-11-24T08:30:06,576 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33555 is added to blk_1073741916_1094 (size=63) 2024-11-24T08:30:06,577 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testCompactedBulkLoadedFiles,,1732437006530.d8f62349b1f39c8b7fcb4535307a9738.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T08:30:06,580 INFO [StoreOpener-d8f62349b1f39c8b7fcb4535307a9738-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region d8f62349b1f39c8b7fcb4535307a9738 2024-11-24T08:30:06,582 INFO [StoreOpener-d8f62349b1f39c8b7fcb4535307a9738-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region d8f62349b1f39c8b7fcb4535307a9738 columnFamilyName a 2024-11-24T08:30:06,582 DEBUG [StoreOpener-d8f62349b1f39c8b7fcb4535307a9738-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:30:06,582 INFO [StoreOpener-d8f62349b1f39c8b7fcb4535307a9738-1 {}] regionserver.HStore(327): Store=d8f62349b1f39c8b7fcb4535307a9738/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T08:30:06,583 INFO [StoreOpener-d8f62349b1f39c8b7fcb4535307a9738-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region d8f62349b1f39c8b7fcb4535307a9738 2024-11-24T08:30:06,584 INFO [StoreOpener-d8f62349b1f39c8b7fcb4535307a9738-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region d8f62349b1f39c8b7fcb4535307a9738 columnFamilyName b 2024-11-24T08:30:06,584 DEBUG [StoreOpener-d8f62349b1f39c8b7fcb4535307a9738-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:30:06,584 INFO [StoreOpener-d8f62349b1f39c8b7fcb4535307a9738-1 {}] regionserver.HStore(327): Store=d8f62349b1f39c8b7fcb4535307a9738/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T08:30:06,584 INFO [StoreOpener-d8f62349b1f39c8b7fcb4535307a9738-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region d8f62349b1f39c8b7fcb4535307a9738 2024-11-24T08:30:06,586 INFO [StoreOpener-d8f62349b1f39c8b7fcb4535307a9738-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region d8f62349b1f39c8b7fcb4535307a9738 columnFamilyName c 2024-11-24T08:30:06,586 DEBUG [StoreOpener-d8f62349b1f39c8b7fcb4535307a9738-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:30:06,586 INFO [StoreOpener-d8f62349b1f39c8b7fcb4535307a9738-1 {}] regionserver.HStore(327): Store=d8f62349b1f39c8b7fcb4535307a9738/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T08:30:06,587 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for d8f62349b1f39c8b7fcb4535307a9738 2024-11-24T08:30:06,588 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46165/hbase/data/default/testCompactedBulkLoadedFiles/d8f62349b1f39c8b7fcb4535307a9738 2024-11-24T08:30:06,588 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46165/hbase/data/default/testCompactedBulkLoadedFiles/d8f62349b1f39c8b7fcb4535307a9738 2024-11-24T08:30:06,589 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for d8f62349b1f39c8b7fcb4535307a9738 2024-11-24T08:30:06,589 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for d8f62349b1f39c8b7fcb4535307a9738 2024-11-24T08:30:06,590 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testCompactedBulkLoadedFiles descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-24T08:30:06,591 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for d8f62349b1f39c8b7fcb4535307a9738 2024-11-24T08:30:06,593 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:46165/hbase/data/default/testCompactedBulkLoadedFiles/d8f62349b1f39c8b7fcb4535307a9738/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-24T08:30:06,594 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened d8f62349b1f39c8b7fcb4535307a9738; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=63935955, jitterRate=-0.04728002846240997}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-24T08:30:06,595 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for d8f62349b1f39c8b7fcb4535307a9738: Writing region info on filesystem at 1732437006577Initializing all the Stores at 1732437006578 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732437006578Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732437006580 (+2 ms)Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732437006580Cleaning up temporary data from old regions at 1732437006589 (+9 ms)Region opened successfully at 1732437006595 (+6 ms) 2024-11-24T08:30:06,595 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing d8f62349b1f39c8b7fcb4535307a9738, disabling compactions & flushes 2024-11-24T08:30:06,595 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testCompactedBulkLoadedFiles,,1732437006530.d8f62349b1f39c8b7fcb4535307a9738. 2024-11-24T08:30:06,595 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testCompactedBulkLoadedFiles,,1732437006530.d8f62349b1f39c8b7fcb4535307a9738. 2024-11-24T08:30:06,595 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testCompactedBulkLoadedFiles,,1732437006530.d8f62349b1f39c8b7fcb4535307a9738. after waiting 0 ms 2024-11-24T08:30:06,595 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testCompactedBulkLoadedFiles,,1732437006530.d8f62349b1f39c8b7fcb4535307a9738. 2024-11-24T08:30:06,596 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testCompactedBulkLoadedFiles,,1732437006530.d8f62349b1f39c8b7fcb4535307a9738. 2024-11-24T08:30:06,596 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for d8f62349b1f39c8b7fcb4535307a9738: Waiting for close lock at 1732437006595Disabling compacts and flushes for region at 1732437006595Disabling writes for close at 1732437006595Writing region close event to WAL at 1732437006596 (+1 ms)Closed at 1732437006596 2024-11-24T08:30:06,603 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40533 is added to blk_1073741915_1093 (size=93) 2024-11-24T08:30:06,603 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33555 is added to blk_1073741915_1093 (size=93) 2024-11-24T08:30:06,603 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34459 is added to blk_1073741915_1093 (size=93) 2024-11-24T08:30:06,606 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-11-24T08:30:06,606 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-28817925:(num 1732437006535) 2024-11-24T08:30:06,606 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-24T08:30:06,608 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:46165/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1732437006529, archiveDir=hdfs://localhost:46165/hbase/oldWALs, maxLogs=32 2024-11-24T08:30:06,621 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1732437006529/wal.1732437006608, exclude list is [], retry=0 2024-11-24T08:30:06,624 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:34459,DS-6b4d1656-f49b-4a16-be49-2eb682c16e11,DISK] 2024-11-24T08:30:06,625 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33555,DS-8d97c3e2-1525-418c-a970-00a7157a0d31,DISK] 2024-11-24T08:30:06,625 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40533,DS-5eeb9666-6dd5-4a48-9fb1-3f9aff1acdca,DISK] 2024-11-24T08:30:06,629 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1732437006529/wal.1732437006608 2024-11-24T08:30:06,634 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:44131:44131),(127.0.0.1/127.0.0.1:39357:39357),(127.0.0.1/127.0.0.1:42673:42673)] 2024-11-24T08:30:06,634 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => d8f62349b1f39c8b7fcb4535307a9738, NAME => 'testCompactedBulkLoadedFiles,,1732437006530.d8f62349b1f39c8b7fcb4535307a9738.', STARTKEY => '', ENDKEY => ''} 2024-11-24T08:30:06,634 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testCompactedBulkLoadedFiles,,1732437006530.d8f62349b1f39c8b7fcb4535307a9738.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T08:30:06,635 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for d8f62349b1f39c8b7fcb4535307a9738 2024-11-24T08:30:06,635 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for d8f62349b1f39c8b7fcb4535307a9738 2024-11-24T08:30:06,636 INFO [StoreOpener-d8f62349b1f39c8b7fcb4535307a9738-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region d8f62349b1f39c8b7fcb4535307a9738 2024-11-24T08:30:06,637 INFO [StoreOpener-d8f62349b1f39c8b7fcb4535307a9738-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region d8f62349b1f39c8b7fcb4535307a9738 columnFamilyName a 2024-11-24T08:30:06,637 DEBUG [StoreOpener-d8f62349b1f39c8b7fcb4535307a9738-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:30:06,638 INFO [StoreOpener-d8f62349b1f39c8b7fcb4535307a9738-1 {}] regionserver.HStore(327): Store=d8f62349b1f39c8b7fcb4535307a9738/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T08:30:06,638 INFO [StoreOpener-d8f62349b1f39c8b7fcb4535307a9738-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region d8f62349b1f39c8b7fcb4535307a9738 2024-11-24T08:30:06,639 INFO [StoreOpener-d8f62349b1f39c8b7fcb4535307a9738-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region d8f62349b1f39c8b7fcb4535307a9738 columnFamilyName b 2024-11-24T08:30:06,639 DEBUG [StoreOpener-d8f62349b1f39c8b7fcb4535307a9738-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:30:06,640 INFO [StoreOpener-d8f62349b1f39c8b7fcb4535307a9738-1 {}] regionserver.HStore(327): Store=d8f62349b1f39c8b7fcb4535307a9738/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T08:30:06,640 INFO [StoreOpener-d8f62349b1f39c8b7fcb4535307a9738-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region d8f62349b1f39c8b7fcb4535307a9738 2024-11-24T08:30:06,641 INFO [StoreOpener-d8f62349b1f39c8b7fcb4535307a9738-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region d8f62349b1f39c8b7fcb4535307a9738 columnFamilyName c 2024-11-24T08:30:06,641 DEBUG [StoreOpener-d8f62349b1f39c8b7fcb4535307a9738-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:30:06,641 INFO [StoreOpener-d8f62349b1f39c8b7fcb4535307a9738-1 {}] regionserver.HStore(327): Store=d8f62349b1f39c8b7fcb4535307a9738/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T08:30:06,641 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for d8f62349b1f39c8b7fcb4535307a9738 2024-11-24T08:30:06,642 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46165/hbase/data/default/testCompactedBulkLoadedFiles/d8f62349b1f39c8b7fcb4535307a9738 2024-11-24T08:30:06,643 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46165/hbase/data/default/testCompactedBulkLoadedFiles/d8f62349b1f39c8b7fcb4535307a9738 2024-11-24T08:30:06,644 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for d8f62349b1f39c8b7fcb4535307a9738 2024-11-24T08:30:06,644 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for d8f62349b1f39c8b7fcb4535307a9738 2024-11-24T08:30:06,645 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testCompactedBulkLoadedFiles descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-24T08:30:06,646 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for d8f62349b1f39c8b7fcb4535307a9738 2024-11-24T08:30:06,647 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened d8f62349b1f39c8b7fcb4535307a9738; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=64629782, jitterRate=-0.036941200494766235}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-24T08:30:06,648 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for d8f62349b1f39c8b7fcb4535307a9738: Writing region info on filesystem at 1732437006635Initializing all the Stores at 1732437006636 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732437006636Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732437006636Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732437006636Cleaning up temporary data from old regions at 1732437006644 (+8 ms)Region opened successfully at 1732437006648 (+4 ms) 2024-11-24T08:30:06,653 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46165/hbase/testCompactedBulkLoadedFiles/hfile0 is 32, key is 000/a:a/1732437006652/Put/seqid=0 2024-11-24T08:30:06,667 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40533 is added to blk_1073741918_1096 (size=4875) 2024-11-24T08:30:06,668 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33555 is added to blk_1073741918_1096 (size=4875) 2024-11-24T08:30:06,668 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34459 is added to blk_1073741918_1096 (size=4875) 2024-11-24T08:30:06,671 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46165/hbase/testCompactedBulkLoadedFiles/hfile1 is 32, key is 100/a:a/1732437006671/Put/seqid=0 2024-11-24T08:30:06,682 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34459 is added to blk_1073741919_1097 (size=4875) 2024-11-24T08:30:06,683 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40533 is added to blk_1073741919_1097 (size=4875) 2024-11-24T08:30:06,689 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33555 is added to blk_1073741919_1097 (size=4875) 2024-11-24T08:30:06,691 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46165/hbase/testCompactedBulkLoadedFiles/hfile2 is 32, key is 200/a:a/1732437006691/Put/seqid=0 2024-11-24T08:30:06,703 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34459 is added to blk_1073741920_1098 (size=4875) 2024-11-24T08:30:06,703 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33555 is added to blk_1073741920_1098 (size=4875) 2024-11-24T08:30:06,703 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40533 is added to blk_1073741920_1098 (size=4875) 2024-11-24T08:30:06,704 INFO [Time-limited test {}] regionserver.HStore(614): Validating hfile at hdfs://localhost:46165/hbase/testCompactedBulkLoadedFiles/hfile0 for inclusion in d8f62349b1f39c8b7fcb4535307a9738/a 2024-11-24T08:30:06,708 DEBUG [Time-limited test {}] regionserver.HStore(626): HFile bounds: first=000 last=050 2024-11-24T08:30:06,708 DEBUG [Time-limited test {}] regionserver.HStore(628): Region bounds: first= last= 2024-11-24T08:30:06,708 INFO [Time-limited test {}] regionserver.HStore(614): Validating hfile at hdfs://localhost:46165/hbase/testCompactedBulkLoadedFiles/hfile1 for inclusion in d8f62349b1f39c8b7fcb4535307a9738/a 2024-11-24T08:30:06,712 DEBUG [Time-limited test {}] regionserver.HStore(626): HFile bounds: first=100 last=150 2024-11-24T08:30:06,712 DEBUG [Time-limited test {}] regionserver.HStore(628): Region bounds: first= last= 2024-11-24T08:30:06,712 INFO [Time-limited test {}] regionserver.HStore(614): Validating hfile at hdfs://localhost:46165/hbase/testCompactedBulkLoadedFiles/hfile2 for inclusion in d8f62349b1f39c8b7fcb4535307a9738/a 2024-11-24T08:30:06,715 DEBUG [Time-limited test {}] regionserver.HStore(626): HFile bounds: first=200 last=250 2024-11-24T08:30:06,715 DEBUG [Time-limited test {}] regionserver.HStore(628): Region bounds: first= last= 2024-11-24T08:30:06,716 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing d8f62349b1f39c8b7fcb4535307a9738 3/3 column families, dataSize=51 B heapSize=896 B 2024-11-24T08:30:06,739 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46165/hbase/data/default/testCompactedBulkLoadedFiles/d8f62349b1f39c8b7fcb4535307a9738/.tmp/a/4c92fbcb2be0421a97828a619fffa264 is 55, key is testCompactedBulkLoadedFiles/a:a/1732437006649/Put/seqid=0 2024-11-24T08:30:06,750 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33555 is added to blk_1073741921_1099 (size=5107) 2024-11-24T08:30:06,750 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40533 is added to blk_1073741921_1099 (size=5107) 2024-11-24T08:30:06,751 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34459 is added to blk_1073741921_1099 (size=5107) 2024-11-24T08:30:06,752 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51 B at sequenceid=4 (bloomFilter=true), to=hdfs://localhost:46165/hbase/data/default/testCompactedBulkLoadedFiles/d8f62349b1f39c8b7fcb4535307a9738/.tmp/a/4c92fbcb2be0421a97828a619fffa264 2024-11-24T08:30:06,758 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46165/hbase/data/default/testCompactedBulkLoadedFiles/d8f62349b1f39c8b7fcb4535307a9738/.tmp/a/4c92fbcb2be0421a97828a619fffa264 as hdfs://localhost:46165/hbase/data/default/testCompactedBulkLoadedFiles/d8f62349b1f39c8b7fcb4535307a9738/a/4c92fbcb2be0421a97828a619fffa264 2024-11-24T08:30:06,764 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46165/hbase/data/default/testCompactedBulkLoadedFiles/d8f62349b1f39c8b7fcb4535307a9738/a/4c92fbcb2be0421a97828a619fffa264, entries=1, sequenceid=4, filesize=5.0 K 2024-11-24T08:30:06,765 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~51 B/51, heapSize ~368 B/368, currentSize=0 B/0 for d8f62349b1f39c8b7fcb4535307a9738 in 50ms, sequenceid=4, compaction requested=false 2024-11-24T08:30:06,765 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for d8f62349b1f39c8b7fcb4535307a9738: 2024-11-24T08:30:06,766 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46165/hbase/testCompactedBulkLoadedFiles/hfile0 as hdfs://localhost:46165/hbase/data/default/testCompactedBulkLoadedFiles/d8f62349b1f39c8b7fcb4535307a9738/a/71bce9f5e33044e9a40ea3b480469e96_SeqId_4_ 2024-11-24T08:30:06,767 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46165/hbase/testCompactedBulkLoadedFiles/hfile1 as hdfs://localhost:46165/hbase/data/default/testCompactedBulkLoadedFiles/d8f62349b1f39c8b7fcb4535307a9738/a/acab3e90805749cda923576f3b8480a9_SeqId_4_ 2024-11-24T08:30:06,768 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46165/hbase/testCompactedBulkLoadedFiles/hfile2 as hdfs://localhost:46165/hbase/data/default/testCompactedBulkLoadedFiles/d8f62349b1f39c8b7fcb4535307a9738/a/e05a3a3ccef443a88c55b00519c79b04_SeqId_4_ 2024-11-24T08:30:06,769 INFO [Time-limited test {}] regionserver.HStore(700): Loaded HFile hdfs://localhost:46165/hbase/testCompactedBulkLoadedFiles/hfile0 into d8f62349b1f39c8b7fcb4535307a9738/a as hdfs://localhost:46165/hbase/data/default/testCompactedBulkLoadedFiles/d8f62349b1f39c8b7fcb4535307a9738/a/71bce9f5e33044e9a40ea3b480469e96_SeqId_4_ - updating store file list. 2024-11-24T08:30:06,773 DEBUG [Time-limited test {}] regionserver.HStoreFile(483): HFile Bloom filter type for 71bce9f5e33044e9a40ea3b480469e96_SeqId_4_: NONE, but ROW specified in column family configuration 2024-11-24T08:30:06,773 INFO [Time-limited test {}] regionserver.HStore(722): Loaded HFile hdfs://localhost:46165/hbase/data/default/testCompactedBulkLoadedFiles/d8f62349b1f39c8b7fcb4535307a9738/a/71bce9f5e33044e9a40ea3b480469e96_SeqId_4_ into d8f62349b1f39c8b7fcb4535307a9738/a 2024-11-24T08:30:06,773 INFO [Time-limited test {}] regionserver.HStore(706): Successfully loaded hdfs://localhost:46165/hbase/testCompactedBulkLoadedFiles/hfile0 into d8f62349b1f39c8b7fcb4535307a9738/a (new location: hdfs://localhost:46165/hbase/data/default/testCompactedBulkLoadedFiles/d8f62349b1f39c8b7fcb4535307a9738/a/71bce9f5e33044e9a40ea3b480469e96_SeqId_4_) 2024-11-24T08:30:06,774 INFO [Time-limited test {}] regionserver.HStore(700): Loaded HFile hdfs://localhost:46165/hbase/testCompactedBulkLoadedFiles/hfile1 into d8f62349b1f39c8b7fcb4535307a9738/a as hdfs://localhost:46165/hbase/data/default/testCompactedBulkLoadedFiles/d8f62349b1f39c8b7fcb4535307a9738/a/acab3e90805749cda923576f3b8480a9_SeqId_4_ - updating store file list. 2024-11-24T08:30:06,778 DEBUG [Time-limited test {}] regionserver.HStoreFile(483): HFile Bloom filter type for acab3e90805749cda923576f3b8480a9_SeqId_4_: NONE, but ROW specified in column family configuration 2024-11-24T08:30:06,778 INFO [Time-limited test {}] regionserver.HStore(722): Loaded HFile hdfs://localhost:46165/hbase/data/default/testCompactedBulkLoadedFiles/d8f62349b1f39c8b7fcb4535307a9738/a/acab3e90805749cda923576f3b8480a9_SeqId_4_ into d8f62349b1f39c8b7fcb4535307a9738/a 2024-11-24T08:30:06,778 INFO [Time-limited test {}] regionserver.HStore(706): Successfully loaded hdfs://localhost:46165/hbase/testCompactedBulkLoadedFiles/hfile1 into d8f62349b1f39c8b7fcb4535307a9738/a (new location: hdfs://localhost:46165/hbase/data/default/testCompactedBulkLoadedFiles/d8f62349b1f39c8b7fcb4535307a9738/a/acab3e90805749cda923576f3b8480a9_SeqId_4_) 2024-11-24T08:30:06,779 INFO [Time-limited test {}] regionserver.HStore(700): Loaded HFile hdfs://localhost:46165/hbase/testCompactedBulkLoadedFiles/hfile2 into d8f62349b1f39c8b7fcb4535307a9738/a as hdfs://localhost:46165/hbase/data/default/testCompactedBulkLoadedFiles/d8f62349b1f39c8b7fcb4535307a9738/a/e05a3a3ccef443a88c55b00519c79b04_SeqId_4_ - updating store file list. 2024-11-24T08:30:06,782 DEBUG [Time-limited test {}] regionserver.HStoreFile(483): HFile Bloom filter type for e05a3a3ccef443a88c55b00519c79b04_SeqId_4_: NONE, but ROW specified in column family configuration 2024-11-24T08:30:06,782 INFO [Time-limited test {}] regionserver.HStore(722): Loaded HFile hdfs://localhost:46165/hbase/data/default/testCompactedBulkLoadedFiles/d8f62349b1f39c8b7fcb4535307a9738/a/e05a3a3ccef443a88c55b00519c79b04_SeqId_4_ into d8f62349b1f39c8b7fcb4535307a9738/a 2024-11-24T08:30:06,782 INFO [Time-limited test {}] regionserver.HStore(706): Successfully loaded hdfs://localhost:46165/hbase/testCompactedBulkLoadedFiles/hfile2 into d8f62349b1f39c8b7fcb4535307a9738/a (new location: hdfs://localhost:46165/hbase/data/default/testCompactedBulkLoadedFiles/d8f62349b1f39c8b7fcb4535307a9738/a/e05a3a3ccef443a88c55b00519c79b04_SeqId_4_) 2024-11-24T08:30:06,790 DEBUG [Time-limited test {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-24T08:30:06,790 DEBUG [Time-limited test {}] regionserver.HStore(1541): d8f62349b1f39c8b7fcb4535307a9738/a is initiating major compaction (all files) 2024-11-24T08:30:06,790 INFO [Time-limited test {}] regionserver.HRegion(2416): Starting compaction of d8f62349b1f39c8b7fcb4535307a9738/a in testCompactedBulkLoadedFiles,,1732437006530.d8f62349b1f39c8b7fcb4535307a9738. 2024-11-24T08:30:06,790 INFO [Time-limited test {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46165/hbase/data/default/testCompactedBulkLoadedFiles/d8f62349b1f39c8b7fcb4535307a9738/a/4c92fbcb2be0421a97828a619fffa264, hdfs://localhost:46165/hbase/data/default/testCompactedBulkLoadedFiles/d8f62349b1f39c8b7fcb4535307a9738/a/71bce9f5e33044e9a40ea3b480469e96_SeqId_4_, hdfs://localhost:46165/hbase/data/default/testCompactedBulkLoadedFiles/d8f62349b1f39c8b7fcb4535307a9738/a/acab3e90805749cda923576f3b8480a9_SeqId_4_, hdfs://localhost:46165/hbase/data/default/testCompactedBulkLoadedFiles/d8f62349b1f39c8b7fcb4535307a9738/a/e05a3a3ccef443a88c55b00519c79b04_SeqId_4_] into tmpdir=hdfs://localhost:46165/hbase/data/default/testCompactedBulkLoadedFiles/d8f62349b1f39c8b7fcb4535307a9738/.tmp, totalSize=19.3 K 2024-11-24T08:30:06,791 DEBUG [Time-limited test {}] compactions.Compactor(225): Compacting 4c92fbcb2be0421a97828a619fffa264, keycount=1, bloomtype=ROW, size=5.0 K, encoding=NONE, compression=NONE, seqNum=4, earliestPutTs=1732437006649 2024-11-24T08:30:06,791 DEBUG [Time-limited test {}] compactions.Compactor(225): Compacting 71bce9f5e33044e9a40ea3b480469e96_SeqId_4_, keycount=10, bloomtype=NONE, size=4.8 K, encoding=NONE, compression=NONE, seqNum=4, earliestPutTs=-9223372036854775808 2024-11-24T08:30:06,792 DEBUG [Time-limited test {}] compactions.Compactor(225): Compacting acab3e90805749cda923576f3b8480a9_SeqId_4_, keycount=10, bloomtype=NONE, size=4.8 K, encoding=NONE, compression=NONE, seqNum=4, earliestPutTs=-9223372036854775808 2024-11-24T08:30:06,792 DEBUG [Time-limited test {}] compactions.Compactor(225): Compacting e05a3a3ccef443a88c55b00519c79b04_SeqId_4_, keycount=10, bloomtype=NONE, size=4.8 K, encoding=NONE, compression=NONE, seqNum=4, earliestPutTs=-9223372036854775808 2024-11-24T08:30:06,803 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46165/hbase/data/default/testCompactedBulkLoadedFiles/d8f62349b1f39c8b7fcb4535307a9738/.tmp/a/6d84d213f88b431581c782494710c803 is 55, key is testCompactedBulkLoadedFiles/a:a/1732437006649/Put/seqid=0 2024-11-24T08:30:06,810 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40533 is added to blk_1073741922_1100 (size=6154) 2024-11-24T08:30:06,810 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34459 is added to blk_1073741922_1100 (size=6154) 2024-11-24T08:30:06,811 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33555 is added to blk_1073741922_1100 (size=6154) 2024-11-24T08:30:06,819 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46165/hbase/data/default/testCompactedBulkLoadedFiles/d8f62349b1f39c8b7fcb4535307a9738/.tmp/a/6d84d213f88b431581c782494710c803 as hdfs://localhost:46165/hbase/data/default/testCompactedBulkLoadedFiles/d8f62349b1f39c8b7fcb4535307a9738/a/6d84d213f88b431581c782494710c803 2024-11-24T08:30:06,826 INFO [Time-limited test {}] regionserver.HStore(1337): Completed major compaction of 4 (all) file(s) in d8f62349b1f39c8b7fcb4535307a9738/a of d8f62349b1f39c8b7fcb4535307a9738 into 6d84d213f88b431581c782494710c803(size=6.0 K), total size for store is 6.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T08:30:06,826 DEBUG [Time-limited test {}] regionserver.HRegion(2446): Compaction status journal for d8f62349b1f39c8b7fcb4535307a9738: 2024-11-24T08:30:06,827 DEBUG [Time-limited test {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 0 store files, 0 compacting, 0 eligible, 16 blocking 2024-11-24T08:30:06,827 DEBUG [Time-limited test {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 0 store files, 0 compacting, 0 eligible, 16 blocking 2024-11-24T08:30:06,860 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:46165/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1732437006529/wal.1732437006608, size=0 (0bytes) 2024-11-24T08:30:06,860 WARN [Time-limited test {}] wal.WALSplitter(453): File hdfs://localhost:46165/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1732437006529/wal.1732437006608 might be still open, length is 0 2024-11-24T08:30:06,861 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:46165/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1732437006529/wal.1732437006608 2024-11-24T08:30:06,861 WARN [IPC Server handler 3 on default port 46165 {}] namenode.FSNamesystem(3854): DIR* NameSystem.internalReleaseLease: File /hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1732437006529/wal.1732437006608 has not been closed. Lease recovery is in progress. RecoveryId = 1101 for block blk_1073741917_1095 2024-11-24T08:30:06,862 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=0 on file=hdfs://localhost:46165/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1732437006529/wal.1732437006608 after 1ms 2024-11-24T08:30:07,257 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testDatalossWhenInputError 2024-11-24T08:30:07,257 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testDatalossWhenInputError Metrics about Tables on a single HBase RegionServer 2024-11-24T08:30:07,258 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testCompactedBulkLoadedFiles 2024-11-24T08:30:07,258 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testCompactedBulkLoadedFiles Metrics about Tables on a single HBase RegionServer 2024-11-24T08:30:08,750 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1541013681_22 at /127.0.0.1:50700 [Receiving block BP-1907186785-172.17.0.2-1732436973683:blk_1073741917_1095] {}] datanode.DataXceiver(331): 127.0.0.1:34459:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:50700 dst: /127.0.0.1:34459 java.io.InterruptedIOException: Interrupted while waiting for IO on channel java.nio.channels.SocketChannel[connected local=localhost/127.0.0.1:34459 remote=/127.0.0.1:50700]. Total timeout mills is 60000, 58077 millis timeout left. at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:350) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?] at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?] at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?] at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?] at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?] at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T08:30:08,750 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1541013681_22 at /127.0.0.1:35876 [Receiving block BP-1907186785-172.17.0.2-1732436973683:blk_1073741917_1095] {}] datanode.DataXceiver(331): 127.0.0.1:33555:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:35876 dst: /127.0.0.1:33555 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T08:30:08,750 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1541013681_22 at /127.0.0.1:42856 [Receiving block BP-1907186785-172.17.0.2-1732436973683:blk_1073741917_1095] {}] datanode.DataXceiver(331): 127.0.0.1:40533:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:42856 dst: /127.0.0.1:40533 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T08:30:08,753 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34459 is added to blk_1073741917_1101 (size=1368) 2024-11-24T08:30:08,754 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33555 is added to blk_1073741917_1101 (size=1368) 2024-11-24T08:30:08,754 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40533 is added to blk_1073741917_1101 (size=1368) 2024-11-24T08:30:10,862 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=1 on file=hdfs://localhost:46165/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1732437006529/wal.1732437006608 after 4001ms 2024-11-24T08:30:10,866 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:46165/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1732437006529/wal.1732437006608: isRecoveredEdits=false, hasTagCompression=true, hasValueCompression=false, valueCompressionType=null 2024-11-24T08:30:10,866 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:46165/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1732437006529/wal.1732437006608 took 4006ms 2024-11-24T08:30:10,868 INFO [Time-limited test {}] wal.WALSplitter(493): EOF from hdfs://localhost:46165/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1732437006529/wal.1732437006608; continuing. 2024-11-24T08:30:10,868 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:46165/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1732437006529/wal.1732437006608 so closing down 2024-11-24T08:30:10,868 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-24T08:30:10,869 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000003-wal.1732437006608.temp 2024-11-24T08:30:10,871 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:46165/hbase/data/default/testCompactedBulkLoadedFiles/d8f62349b1f39c8b7fcb4535307a9738/recovered.edits/0000000000000000003-wal.1732437006608.temp 2024-11-24T08:30:10,871 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-11-24T08:30:10,877 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40533 is added to blk_1073741923_1102 (size=635) 2024-11-24T08:30:10,877 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34459 is added to blk_1073741923_1102 (size=635) 2024-11-24T08:30:10,878 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33555 is added to blk_1073741923_1102 (size=635) 2024-11-24T08:30:10,879 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:46165/hbase/data/default/testCompactedBulkLoadedFiles/d8f62349b1f39c8b7fcb4535307a9738/recovered.edits/0000000000000000003-wal.1732437006608.temp (wrote 2 edits, skipped 0 edits in 0 ms) 2024-11-24T08:30:10,880 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:46165/hbase/data/default/testCompactedBulkLoadedFiles/d8f62349b1f39c8b7fcb4535307a9738/recovered.edits/0000000000000000003-wal.1732437006608.temp to hdfs://localhost:46165/hbase/data/default/testCompactedBulkLoadedFiles/d8f62349b1f39c8b7fcb4535307a9738/recovered.edits/0000000000000000008 2024-11-24T08:30:10,880 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 5 edits across 1 Regions in 14 ms; skipped=3; WAL=hdfs://localhost:46165/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1732437006529/wal.1732437006608, size=0, length=0, corrupted=false, cancelled=false 2024-11-24T08:30:10,880 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:46165/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1732437006529/wal.1732437006608, journal: Splitting hdfs://localhost:46165/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1732437006529/wal.1732437006608, size=0 (0bytes) at 1732437006860Finishing writing output for hdfs://localhost:46165/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1732437006529/wal.1732437006608 so closing down at 1732437010868 (+4008 ms)Creating recovered edits writer path=hdfs://localhost:46165/hbase/data/default/testCompactedBulkLoadedFiles/d8f62349b1f39c8b7fcb4535307a9738/recovered.edits/0000000000000000003-wal.1732437006608.temp at 1732437010871 (+3 ms)3 split writer threads finished at 1732437010871Closed recovered edits writer path=hdfs://localhost:46165/hbase/data/default/testCompactedBulkLoadedFiles/d8f62349b1f39c8b7fcb4535307a9738/recovered.edits/0000000000000000003-wal.1732437006608.temp (wrote 2 edits, skipped 0 edits in 0 ms) at 1732437010879 (+8 ms)Rename recovered edits hdfs://localhost:46165/hbase/data/default/testCompactedBulkLoadedFiles/d8f62349b1f39c8b7fcb4535307a9738/recovered.edits/0000000000000000003-wal.1732437006608.temp to hdfs://localhost:46165/hbase/data/default/testCompactedBulkLoadedFiles/d8f62349b1f39c8b7fcb4535307a9738/recovered.edits/0000000000000000008 at 1732437010880 (+1 ms)Processed 5 edits across 1 Regions in 14 ms; skipped=3; WAL=hdfs://localhost:46165/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1732437006529/wal.1732437006608, size=0, length=0, corrupted=false, cancelled=false at 1732437010880 2024-11-24T08:30:10,882 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:46165/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1732437006529/wal.1732437006608 to hdfs://localhost:46165/hbase/oldWALs/wal.1732437006608 2024-11-24T08:30:10,883 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:46165/hbase/data/default/testCompactedBulkLoadedFiles/d8f62349b1f39c8b7fcb4535307a9738/recovered.edits/0000000000000000008 2024-11-24T08:30:10,883 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-24T08:30:10,885 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:46165/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1732437006529, archiveDir=hdfs://localhost:46165/hbase/oldWALs, maxLogs=32 2024-11-24T08:30:10,899 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1732437006529/wal.1732437010885, exclude list is [], retry=0 2024-11-24T08:30:10,901 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40533,DS-5eeb9666-6dd5-4a48-9fb1-3f9aff1acdca,DISK] 2024-11-24T08:30:10,902 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:34459,DS-6b4d1656-f49b-4a16-be49-2eb682c16e11,DISK] 2024-11-24T08:30:10,902 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33555,DS-8d97c3e2-1525-418c-a970-00a7157a0d31,DISK] 2024-11-24T08:30:10,904 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1732437006529/wal.1732437010885 2024-11-24T08:30:10,904 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:42673:42673),(127.0.0.1/127.0.0.1:44131:44131),(127.0.0.1/127.0.0.1:39357:39357)] 2024-11-24T08:30:10,905 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => d8f62349b1f39c8b7fcb4535307a9738, NAME => 'testCompactedBulkLoadedFiles,,1732437006530.d8f62349b1f39c8b7fcb4535307a9738.', STARTKEY => '', ENDKEY => ''} 2024-11-24T08:30:10,905 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testCompactedBulkLoadedFiles,,1732437006530.d8f62349b1f39c8b7fcb4535307a9738.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T08:30:10,905 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for d8f62349b1f39c8b7fcb4535307a9738 2024-11-24T08:30:10,905 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for d8f62349b1f39c8b7fcb4535307a9738 2024-11-24T08:30:10,906 INFO [StoreOpener-d8f62349b1f39c8b7fcb4535307a9738-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region d8f62349b1f39c8b7fcb4535307a9738 2024-11-24T08:30:10,907 INFO [StoreOpener-d8f62349b1f39c8b7fcb4535307a9738-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region d8f62349b1f39c8b7fcb4535307a9738 columnFamilyName a 2024-11-24T08:30:10,907 DEBUG [StoreOpener-d8f62349b1f39c8b7fcb4535307a9738-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:30:10,914 DEBUG [StoreOpener-d8f62349b1f39c8b7fcb4535307a9738-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:46165/hbase/data/default/testCompactedBulkLoadedFiles/d8f62349b1f39c8b7fcb4535307a9738/a/4c92fbcb2be0421a97828a619fffa264 2024-11-24T08:30:10,918 DEBUG [StoreOpener-d8f62349b1f39c8b7fcb4535307a9738-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:46165/hbase/data/default/testCompactedBulkLoadedFiles/d8f62349b1f39c8b7fcb4535307a9738/a/6d84d213f88b431581c782494710c803 2024-11-24T08:30:10,921 DEBUG [StoreFileOpener-d8f62349b1f39c8b7fcb4535307a9738-a-1 {}] regionserver.HStoreFile(483): HFile Bloom filter type for 71bce9f5e33044e9a40ea3b480469e96_SeqId_4_: NONE, but ROW specified in column family configuration 2024-11-24T08:30:10,921 DEBUG [StoreOpener-d8f62349b1f39c8b7fcb4535307a9738-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:46165/hbase/data/default/testCompactedBulkLoadedFiles/d8f62349b1f39c8b7fcb4535307a9738/a/71bce9f5e33044e9a40ea3b480469e96_SeqId_4_ 2024-11-24T08:30:10,925 DEBUG [StoreFileOpener-d8f62349b1f39c8b7fcb4535307a9738-a-1 {}] regionserver.HStoreFile(483): HFile Bloom filter type for acab3e90805749cda923576f3b8480a9_SeqId_4_: NONE, but ROW specified in column family configuration 2024-11-24T08:30:10,925 DEBUG [StoreOpener-d8f62349b1f39c8b7fcb4535307a9738-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:46165/hbase/data/default/testCompactedBulkLoadedFiles/d8f62349b1f39c8b7fcb4535307a9738/a/acab3e90805749cda923576f3b8480a9_SeqId_4_ 2024-11-24T08:30:10,928 DEBUG [StoreFileOpener-d8f62349b1f39c8b7fcb4535307a9738-a-1 {}] regionserver.HStoreFile(483): HFile Bloom filter type for e05a3a3ccef443a88c55b00519c79b04_SeqId_4_: NONE, but ROW specified in column family configuration 2024-11-24T08:30:10,928 DEBUG [StoreOpener-d8f62349b1f39c8b7fcb4535307a9738-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:46165/hbase/data/default/testCompactedBulkLoadedFiles/d8f62349b1f39c8b7fcb4535307a9738/a/e05a3a3ccef443a88c55b00519c79b04_SeqId_4_ 2024-11-24T08:30:10,929 WARN [StoreOpener-d8f62349b1f39c8b7fcb4535307a9738-1 {}] regionserver.StoreEngine(317): Clearing the compacted storefile hdfs://localhost:46165/hbase/data/default/testCompactedBulkLoadedFiles/d8f62349b1f39c8b7fcb4535307a9738/a/4c92fbcb2be0421a97828a619fffa264 from org.apache.hadoop.hbase.regionserver.DefaultStoreEngine@3df32de1 2024-11-24T08:30:10,929 WARN [StoreOpener-d8f62349b1f39c8b7fcb4535307a9738-1 {}] regionserver.StoreEngine(317): Clearing the compacted storefile hdfs://localhost:46165/hbase/data/default/testCompactedBulkLoadedFiles/d8f62349b1f39c8b7fcb4535307a9738/a/71bce9f5e33044e9a40ea3b480469e96_SeqId_4_ from org.apache.hadoop.hbase.regionserver.DefaultStoreEngine@3df32de1 2024-11-24T08:30:10,929 WARN [StoreOpener-d8f62349b1f39c8b7fcb4535307a9738-1 {}] regionserver.StoreEngine(317): Clearing the compacted storefile hdfs://localhost:46165/hbase/data/default/testCompactedBulkLoadedFiles/d8f62349b1f39c8b7fcb4535307a9738/a/acab3e90805749cda923576f3b8480a9_SeqId_4_ from org.apache.hadoop.hbase.regionserver.DefaultStoreEngine@3df32de1 2024-11-24T08:30:10,929 WARN [StoreOpener-d8f62349b1f39c8b7fcb4535307a9738-1 {}] regionserver.StoreEngine(317): Clearing the compacted storefile hdfs://localhost:46165/hbase/data/default/testCompactedBulkLoadedFiles/d8f62349b1f39c8b7fcb4535307a9738/a/e05a3a3ccef443a88c55b00519c79b04_SeqId_4_ from org.apache.hadoop.hbase.regionserver.DefaultStoreEngine@3df32de1 2024-11-24T08:30:10,929 DEBUG [StoreOpener-d8f62349b1f39c8b7fcb4535307a9738-1 {}] regionserver.StoreEngine(327): Moving the files [hdfs://localhost:46165/hbase/data/default/testCompactedBulkLoadedFiles/d8f62349b1f39c8b7fcb4535307a9738/a/4c92fbcb2be0421a97828a619fffa264, hdfs://localhost:46165/hbase/data/default/testCompactedBulkLoadedFiles/d8f62349b1f39c8b7fcb4535307a9738/a/71bce9f5e33044e9a40ea3b480469e96_SeqId_4_, hdfs://localhost:46165/hbase/data/default/testCompactedBulkLoadedFiles/d8f62349b1f39c8b7fcb4535307a9738/a/acab3e90805749cda923576f3b8480a9_SeqId_4_, hdfs://localhost:46165/hbase/data/default/testCompactedBulkLoadedFiles/d8f62349b1f39c8b7fcb4535307a9738/a/e05a3a3ccef443a88c55b00519c79b04_SeqId_4_] to archive 2024-11-24T08:30:10,930 DEBUG [StoreOpener-d8f62349b1f39c8b7fcb4535307a9738-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-24T08:30:10,932 DEBUG [StoreOpener-d8f62349b1f39c8b7fcb4535307a9738-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46165/hbase/data/default/testCompactedBulkLoadedFiles/d8f62349b1f39c8b7fcb4535307a9738/a/4c92fbcb2be0421a97828a619fffa264 to hdfs://localhost:46165/hbase/archive/data/default/testCompactedBulkLoadedFiles/d8f62349b1f39c8b7fcb4535307a9738/a/4c92fbcb2be0421a97828a619fffa264 2024-11-24T08:30:10,933 DEBUG [StoreOpener-d8f62349b1f39c8b7fcb4535307a9738-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46165/hbase/data/default/testCompactedBulkLoadedFiles/d8f62349b1f39c8b7fcb4535307a9738/a/71bce9f5e33044e9a40ea3b480469e96_SeqId_4_ to hdfs://localhost:46165/hbase/archive/data/default/testCompactedBulkLoadedFiles/d8f62349b1f39c8b7fcb4535307a9738/a/71bce9f5e33044e9a40ea3b480469e96_SeqId_4_ 2024-11-24T08:30:10,934 DEBUG [StoreOpener-d8f62349b1f39c8b7fcb4535307a9738-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46165/hbase/data/default/testCompactedBulkLoadedFiles/d8f62349b1f39c8b7fcb4535307a9738/a/acab3e90805749cda923576f3b8480a9_SeqId_4_ to hdfs://localhost:46165/hbase/archive/data/default/testCompactedBulkLoadedFiles/d8f62349b1f39c8b7fcb4535307a9738/a/acab3e90805749cda923576f3b8480a9_SeqId_4_ 2024-11-24T08:30:10,935 DEBUG [StoreOpener-d8f62349b1f39c8b7fcb4535307a9738-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46165/hbase/data/default/testCompactedBulkLoadedFiles/d8f62349b1f39c8b7fcb4535307a9738/a/e05a3a3ccef443a88c55b00519c79b04_SeqId_4_ to hdfs://localhost:46165/hbase/archive/data/default/testCompactedBulkLoadedFiles/d8f62349b1f39c8b7fcb4535307a9738/a/e05a3a3ccef443a88c55b00519c79b04_SeqId_4_ 2024-11-24T08:30:10,935 INFO [StoreOpener-d8f62349b1f39c8b7fcb4535307a9738-1 {}] regionserver.HStore(327): Store=d8f62349b1f39c8b7fcb4535307a9738/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T08:30:10,935 INFO [StoreOpener-d8f62349b1f39c8b7fcb4535307a9738-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region d8f62349b1f39c8b7fcb4535307a9738 2024-11-24T08:30:10,936 INFO [StoreOpener-d8f62349b1f39c8b7fcb4535307a9738-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region d8f62349b1f39c8b7fcb4535307a9738 columnFamilyName b 2024-11-24T08:30:10,936 DEBUG [StoreOpener-d8f62349b1f39c8b7fcb4535307a9738-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:30:10,937 INFO [StoreOpener-d8f62349b1f39c8b7fcb4535307a9738-1 {}] regionserver.HStore(327): Store=d8f62349b1f39c8b7fcb4535307a9738/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T08:30:10,937 INFO [StoreOpener-d8f62349b1f39c8b7fcb4535307a9738-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region d8f62349b1f39c8b7fcb4535307a9738 2024-11-24T08:30:10,937 INFO [StoreOpener-d8f62349b1f39c8b7fcb4535307a9738-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region d8f62349b1f39c8b7fcb4535307a9738 columnFamilyName c 2024-11-24T08:30:10,937 DEBUG [StoreOpener-d8f62349b1f39c8b7fcb4535307a9738-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:30:10,938 INFO [StoreOpener-d8f62349b1f39c8b7fcb4535307a9738-1 {}] regionserver.HStore(327): Store=d8f62349b1f39c8b7fcb4535307a9738/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T08:30:10,938 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for d8f62349b1f39c8b7fcb4535307a9738 2024-11-24T08:30:10,939 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46165/hbase/data/default/testCompactedBulkLoadedFiles/d8f62349b1f39c8b7fcb4535307a9738 2024-11-24T08:30:10,940 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:46165/hbase/data/default/testCompactedBulkLoadedFiles/d8f62349b1f39c8b7fcb4535307a9738 2024-11-24T08:30:10,940 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:46165/hbase/data/default/testCompactedBulkLoadedFiles/d8f62349b1f39c8b7fcb4535307a9738/recovered.edits/0000000000000000008 2024-11-24T08:30:10,943 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:46165/hbase/data/default/testCompactedBulkLoadedFiles/d8f62349b1f39c8b7fcb4535307a9738/recovered.edits/0000000000000000008: isRecoveredEdits=true, hasTagCompression=true, hasValueCompression=false, valueCompressionType=null 2024-11-24T08:30:10,944 DEBUG [Time-limited test {}] regionserver.HRegion(5836): d8f62349b1f39c8b7fcb4535307a9738 : Replaying compaction marker table_name: "testCompactedBulkLoadedFiles" encoded_region_name: "d8f62349b1f39c8b7fcb4535307a9738" family_name: "a" compaction_input: "4c92fbcb2be0421a97828a619fffa264" compaction_input: "71bce9f5e33044e9a40ea3b480469e96_SeqId_4_" compaction_input: "acab3e90805749cda923576f3b8480a9_SeqId_4_" compaction_input: "e05a3a3ccef443a88c55b00519c79b04_SeqId_4_" compaction_output: "6d84d213f88b431581c782494710c803" store_home_dir: "a" region_name: "testCompactedBulkLoadedFiles,,1732437006530.d8f62349b1f39c8b7fcb4535307a9738." with seqId=9223372036854775807 and lastReplayedOpenRegionSeqId=-1 2024-11-24T08:30:10,945 DEBUG [Time-limited test {}] regionserver.HStore(1354): Completing compaction from the WAL marker 2024-11-24T08:30:10,945 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 0, skipped 2, firstSequenceIdInLog=3, maxSequenceIdInLog=8, path=hdfs://localhost:46165/hbase/data/default/testCompactedBulkLoadedFiles/d8f62349b1f39c8b7fcb4535307a9738/recovered.edits/0000000000000000008 2024-11-24T08:30:10,945 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:46165/hbase/data/default/testCompactedBulkLoadedFiles/d8f62349b1f39c8b7fcb4535307a9738/recovered.edits/0000000000000000008 2024-11-24T08:30:10,946 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for d8f62349b1f39c8b7fcb4535307a9738 2024-11-24T08:30:10,946 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for d8f62349b1f39c8b7fcb4535307a9738 2024-11-24T08:30:10,947 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testCompactedBulkLoadedFiles descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-24T08:30:10,948 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for d8f62349b1f39c8b7fcb4535307a9738 2024-11-24T08:30:10,951 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:46165/hbase/data/default/testCompactedBulkLoadedFiles/d8f62349b1f39c8b7fcb4535307a9738/recovered.edits/8.seqid, newMaxSeqId=8, maxSeqId=1 2024-11-24T08:30:10,952 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened d8f62349b1f39c8b7fcb4535307a9738; next sequenceid=9; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=72047540, jitterRate=0.07359200716018677}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-24T08:30:10,952 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for d8f62349b1f39c8b7fcb4535307a9738: Writing region info on filesystem at 1732437010905Initializing all the Stores at 1732437010906 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732437010906Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732437010906Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732437010906Cleaning up temporary data from old regions at 1732437010946 (+40 ms)Region opened successfully at 1732437010952 (+6 ms) 2024-11-24T08:30:10,955 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing d8f62349b1f39c8b7fcb4535307a9738, disabling compactions & flushes 2024-11-24T08:30:10,955 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testCompactedBulkLoadedFiles,,1732437006530.d8f62349b1f39c8b7fcb4535307a9738. 2024-11-24T08:30:10,955 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testCompactedBulkLoadedFiles,,1732437006530.d8f62349b1f39c8b7fcb4535307a9738. 2024-11-24T08:30:10,955 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testCompactedBulkLoadedFiles,,1732437006530.d8f62349b1f39c8b7fcb4535307a9738. after waiting 0 ms 2024-11-24T08:30:10,955 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testCompactedBulkLoadedFiles,,1732437006530.d8f62349b1f39c8b7fcb4535307a9738. 2024-11-24T08:30:10,956 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testCompactedBulkLoadedFiles,,1732437006530.d8f62349b1f39c8b7fcb4535307a9738. 2024-11-24T08:30:10,956 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for d8f62349b1f39c8b7fcb4535307a9738: Waiting for close lock at 1732437010954Disabling compacts and flushes for region at 1732437010954Disabling writes for close at 1732437010955 (+1 ms)Writing region close event to WAL at 1732437010956 (+1 ms)Closed at 1732437010956 2024-11-24T08:30:10,958 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40533 is added to blk_1073741924_1103 (size=93) 2024-11-24T08:30:10,958 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33555 is added to blk_1073741924_1103 (size=93) 2024-11-24T08:30:10,959 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34459 is added to blk_1073741924_1103 (size=93) 2024-11-24T08:30:10,961 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-11-24T08:30:10,961 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL wal:(num 1732437010885) 2024-11-24T08:30:10,978 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestAsyncWALReplayCompressed#testCompactedBulkLoadedFiles Thread=437 (was 425) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_657624370_22 at /127.0.0.1:48180 [Waiting for operation #11] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-26-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_657624370_22 at /127.0.0.1:39046 [Waiting for operation #7] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Client (1987708253) connection to localhost/127.0.0.1:46165 from jenkinstestCompactedBulkLoadedFiles java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) Potentially hanging thread: AsyncFSWAL-26-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: LeaseRenewer:jenkinstestCompactedBulkLoadedFiles@localhost:46165 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-26-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=1344 (was 1262) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=202 (was 210), ProcessCount=11 (was 11), AvailableMemoryMB=6238 (was 6172) - AvailableMemoryMB LEAK? - 2024-11-24T08:30:10,978 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1344 is superior to 1024 2024-11-24T08:30:10,991 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestAsyncWALReplayCompressed#testReplayEditsWrittenViaHRegion Thread=437, OpenFileDescriptor=1344, MaxFileDescriptor=1048576, SystemLoadAverage=202, ProcessCount=11, AvailableMemoryMB=6237 2024-11-24T08:30:10,992 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1344 is superior to 1024 2024-11-24T08:30:11,007 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-24T08:30:11,010 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-24T08:30:11,010 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-24T08:30:11,013 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-91137562, suffix=, logDir=hdfs://localhost:46165/hbase/WALs/hregion-91137562, archiveDir=hdfs://localhost:46165/hbase/oldWALs, maxLogs=32 2024-11-24T08:30:11,033 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-91137562/hregion-91137562.1732437011013, exclude list is [], retry=0 2024-11-24T08:30:11,036 DEBUG [AsyncFSWAL-28-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40533,DS-5eeb9666-6dd5-4a48-9fb1-3f9aff1acdca,DISK] 2024-11-24T08:30:11,037 DEBUG [AsyncFSWAL-28-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:34459,DS-6b4d1656-f49b-4a16-be49-2eb682c16e11,DISK] 2024-11-24T08:30:11,037 DEBUG [AsyncFSWAL-28-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33555,DS-8d97c3e2-1525-418c-a970-00a7157a0d31,DISK] 2024-11-24T08:30:11,040 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-91137562/hregion-91137562.1732437011013 2024-11-24T08:30:11,040 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:42673:42673),(127.0.0.1/127.0.0.1:44131:44131),(127.0.0.1/127.0.0.1:39357:39357)] 2024-11-24T08:30:11,040 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => d677ce833c832574d4ab35c1b606b096, NAME => 'testReplayEditsWrittenViaHRegion,,1732437011008.d677ce833c832574d4ab35c1b606b096.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testReplayEditsWrittenViaHRegion', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:46165/hbase 2024-11-24T08:30:11,054 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33555 is added to blk_1073741926_1105 (size=67) 2024-11-24T08:30:11,054 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40533 is added to blk_1073741926_1105 (size=67) 2024-11-24T08:30:11,055 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34459 is added to blk_1073741926_1105 (size=67) 2024-11-24T08:30:11,055 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenViaHRegion,,1732437011008.d677ce833c832574d4ab35c1b606b096.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T08:30:11,057 INFO [StoreOpener-d677ce833c832574d4ab35c1b606b096-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region d677ce833c832574d4ab35c1b606b096 2024-11-24T08:30:11,058 INFO [StoreOpener-d677ce833c832574d4ab35c1b606b096-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region d677ce833c832574d4ab35c1b606b096 columnFamilyName a 2024-11-24T08:30:11,058 DEBUG [StoreOpener-d677ce833c832574d4ab35c1b606b096-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:30:11,059 INFO [StoreOpener-d677ce833c832574d4ab35c1b606b096-1 {}] regionserver.HStore(327): Store=d677ce833c832574d4ab35c1b606b096/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T08:30:11,059 INFO [StoreOpener-d677ce833c832574d4ab35c1b606b096-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region d677ce833c832574d4ab35c1b606b096 2024-11-24T08:30:11,060 INFO [StoreOpener-d677ce833c832574d4ab35c1b606b096-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region d677ce833c832574d4ab35c1b606b096 columnFamilyName b 2024-11-24T08:30:11,060 DEBUG [StoreOpener-d677ce833c832574d4ab35c1b606b096-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:30:11,061 INFO [StoreOpener-d677ce833c832574d4ab35c1b606b096-1 {}] regionserver.HStore(327): Store=d677ce833c832574d4ab35c1b606b096/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T08:30:11,061 INFO [StoreOpener-d677ce833c832574d4ab35c1b606b096-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region d677ce833c832574d4ab35c1b606b096 2024-11-24T08:30:11,062 INFO [StoreOpener-d677ce833c832574d4ab35c1b606b096-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region d677ce833c832574d4ab35c1b606b096 columnFamilyName c 2024-11-24T08:30:11,062 DEBUG [StoreOpener-d677ce833c832574d4ab35c1b606b096-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:30:11,063 INFO [StoreOpener-d677ce833c832574d4ab35c1b606b096-1 {}] regionserver.HStore(327): Store=d677ce833c832574d4ab35c1b606b096/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T08:30:11,063 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for d677ce833c832574d4ab35c1b606b096 2024-11-24T08:30:11,063 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46165/hbase/data/default/testReplayEditsWrittenViaHRegion/d677ce833c832574d4ab35c1b606b096 2024-11-24T08:30:11,064 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46165/hbase/data/default/testReplayEditsWrittenViaHRegion/d677ce833c832574d4ab35c1b606b096 2024-11-24T08:30:11,065 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for d677ce833c832574d4ab35c1b606b096 2024-11-24T08:30:11,065 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for d677ce833c832574d4ab35c1b606b096 2024-11-24T08:30:11,065 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsWrittenViaHRegion descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-24T08:30:11,067 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for d677ce833c832574d4ab35c1b606b096 2024-11-24T08:30:11,069 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:46165/hbase/data/default/testReplayEditsWrittenViaHRegion/d677ce833c832574d4ab35c1b606b096/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-24T08:30:11,069 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened d677ce833c832574d4ab35c1b606b096; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=72298512, jitterRate=0.0773317813873291}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-24T08:30:11,069 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for d677ce833c832574d4ab35c1b606b096: Writing region info on filesystem at 1732437011055Initializing all the Stores at 1732437011056 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732437011056Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732437011057 (+1 ms)Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732437011057Cleaning up temporary data from old regions at 1732437011065 (+8 ms)Region opened successfully at 1732437011069 (+4 ms) 2024-11-24T08:30:11,069 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing d677ce833c832574d4ab35c1b606b096, disabling compactions & flushes 2024-11-24T08:30:11,069 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsWrittenViaHRegion,,1732437011008.d677ce833c832574d4ab35c1b606b096. 2024-11-24T08:30:11,069 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsWrittenViaHRegion,,1732437011008.d677ce833c832574d4ab35c1b606b096. 2024-11-24T08:30:11,070 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsWrittenViaHRegion,,1732437011008.d677ce833c832574d4ab35c1b606b096. after waiting 0 ms 2024-11-24T08:30:11,070 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsWrittenViaHRegion,,1732437011008.d677ce833c832574d4ab35c1b606b096. 2024-11-24T08:30:11,070 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsWrittenViaHRegion,,1732437011008.d677ce833c832574d4ab35c1b606b096. 2024-11-24T08:30:11,070 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for d677ce833c832574d4ab35c1b606b096: Waiting for close lock at 1732437011069Disabling compacts and flushes for region at 1732437011069Disabling writes for close at 1732437011070 (+1 ms)Writing region close event to WAL at 1732437011070Closed at 1732437011070 2024-11-24T08:30:11,072 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40533 is added to blk_1073741925_1104 (size=93) 2024-11-24T08:30:11,072 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34459 is added to blk_1073741925_1104 (size=93) 2024-11-24T08:30:11,073 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33555 is added to blk_1073741925_1104 (size=93) 2024-11-24T08:30:11,074 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-11-24T08:30:11,075 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-91137562:(num 1732437011013) 2024-11-24T08:30:11,075 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-24T08:30:11,077 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:46165/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732437011007, archiveDir=hdfs://localhost:46165/hbase/oldWALs, maxLogs=32 2024-11-24T08:30:11,096 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732437011007/wal.1732437011077, exclude list is [], retry=0 2024-11-24T08:30:11,098 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40533,DS-5eeb9666-6dd5-4a48-9fb1-3f9aff1acdca,DISK] 2024-11-24T08:30:11,099 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:34459,DS-6b4d1656-f49b-4a16-be49-2eb682c16e11,DISK] 2024-11-24T08:30:11,099 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33555,DS-8d97c3e2-1525-418c-a970-00a7157a0d31,DISK] 2024-11-24T08:30:11,100 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732437011007/wal.1732437011077 2024-11-24T08:30:11,101 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:42673:42673),(127.0.0.1/127.0.0.1:44131:44131),(127.0.0.1/127.0.0.1:39357:39357)] 2024-11-24T08:30:11,101 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => d677ce833c832574d4ab35c1b606b096, NAME => 'testReplayEditsWrittenViaHRegion,,1732437011008.d677ce833c832574d4ab35c1b606b096.', STARTKEY => '', ENDKEY => ''} 2024-11-24T08:30:11,101 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenViaHRegion,,1732437011008.d677ce833c832574d4ab35c1b606b096.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T08:30:11,101 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for d677ce833c832574d4ab35c1b606b096 2024-11-24T08:30:11,101 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for d677ce833c832574d4ab35c1b606b096 2024-11-24T08:30:11,103 INFO [StoreOpener-d677ce833c832574d4ab35c1b606b096-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region d677ce833c832574d4ab35c1b606b096 2024-11-24T08:30:11,104 INFO [StoreOpener-d677ce833c832574d4ab35c1b606b096-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region d677ce833c832574d4ab35c1b606b096 columnFamilyName a 2024-11-24T08:30:11,104 DEBUG [StoreOpener-d677ce833c832574d4ab35c1b606b096-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:30:11,104 INFO [StoreOpener-d677ce833c832574d4ab35c1b606b096-1 {}] regionserver.HStore(327): Store=d677ce833c832574d4ab35c1b606b096/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T08:30:11,104 INFO [StoreOpener-d677ce833c832574d4ab35c1b606b096-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region d677ce833c832574d4ab35c1b606b096 2024-11-24T08:30:11,105 INFO [StoreOpener-d677ce833c832574d4ab35c1b606b096-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region d677ce833c832574d4ab35c1b606b096 columnFamilyName b 2024-11-24T08:30:11,105 DEBUG [StoreOpener-d677ce833c832574d4ab35c1b606b096-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:30:11,105 INFO [StoreOpener-d677ce833c832574d4ab35c1b606b096-1 {}] regionserver.HStore(327): Store=d677ce833c832574d4ab35c1b606b096/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T08:30:11,105 INFO [StoreOpener-d677ce833c832574d4ab35c1b606b096-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region d677ce833c832574d4ab35c1b606b096 2024-11-24T08:30:11,106 INFO [StoreOpener-d677ce833c832574d4ab35c1b606b096-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region d677ce833c832574d4ab35c1b606b096 columnFamilyName c 2024-11-24T08:30:11,106 DEBUG [StoreOpener-d677ce833c832574d4ab35c1b606b096-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:30:11,107 INFO [StoreOpener-d677ce833c832574d4ab35c1b606b096-1 {}] regionserver.HStore(327): Store=d677ce833c832574d4ab35c1b606b096/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T08:30:11,107 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for d677ce833c832574d4ab35c1b606b096 2024-11-24T08:30:11,108 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46165/hbase/data/default/testReplayEditsWrittenViaHRegion/d677ce833c832574d4ab35c1b606b096 2024-11-24T08:30:11,109 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46165/hbase/data/default/testReplayEditsWrittenViaHRegion/d677ce833c832574d4ab35c1b606b096 2024-11-24T08:30:11,110 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for d677ce833c832574d4ab35c1b606b096 2024-11-24T08:30:11,110 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for d677ce833c832574d4ab35c1b606b096 2024-11-24T08:30:11,110 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsWrittenViaHRegion descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-24T08:30:11,111 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for d677ce833c832574d4ab35c1b606b096 2024-11-24T08:30:11,112 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened d677ce833c832574d4ab35c1b606b096; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=72812265, jitterRate=0.08498729765415192}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-24T08:30:11,112 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for d677ce833c832574d4ab35c1b606b096: Writing region info on filesystem at 1732437011101Initializing all the Stores at 1732437011102 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732437011102Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732437011103 (+1 ms)Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732437011103Cleaning up temporary data from old regions at 1732437011110 (+7 ms)Region opened successfully at 1732437011112 (+2 ms) 2024-11-24T08:30:11,121 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing d677ce833c832574d4ab35c1b606b096 3/3 column families, dataSize=870 B heapSize=2.31 KB 2024-11-24T08:30:11,128 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-24T08:30:11,143 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46165/hbase/data/default/testReplayEditsWrittenViaHRegion/d677ce833c832574d4ab35c1b606b096/.tmp/a/439792cb36bf4b9ca42dbdd87a1a73d9 is 91, key is testReplayEditsWrittenViaHRegion/a:x0/1732437011112/Put/seqid=0 2024-11-24T08:30:11,151 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40533 is added to blk_1073741928_1107 (size=5958) 2024-11-24T08:30:11,151 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34459 is added to blk_1073741928_1107 (size=5958) 2024-11-24T08:30:11,151 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33555 is added to blk_1073741928_1107 (size=5958) 2024-11-24T08:30:11,152 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:46165/hbase/data/default/testReplayEditsWrittenViaHRegion/d677ce833c832574d4ab35c1b606b096/.tmp/a/439792cb36bf4b9ca42dbdd87a1a73d9 2024-11-24T08:30:11,159 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46165/hbase/data/default/testReplayEditsWrittenViaHRegion/d677ce833c832574d4ab35c1b606b096/.tmp/a/439792cb36bf4b9ca42dbdd87a1a73d9 as hdfs://localhost:46165/hbase/data/default/testReplayEditsWrittenViaHRegion/d677ce833c832574d4ab35c1b606b096/a/439792cb36bf4b9ca42dbdd87a1a73d9 2024-11-24T08:30:11,166 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46165/hbase/data/default/testReplayEditsWrittenViaHRegion/d677ce833c832574d4ab35c1b606b096/a/439792cb36bf4b9ca42dbdd87a1a73d9, entries=10, sequenceid=13, filesize=5.8 K 2024-11-24T08:30:11,167 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~870 B/870, heapSize ~1.80 KB/1840, currentSize=0 B/0 for d677ce833c832574d4ab35c1b606b096 in 46ms, sequenceid=13, compaction requested=false 2024-11-24T08:30:11,167 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for d677ce833c832574d4ab35c1b606b096: 2024-11-24T08:30:11,188 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing d677ce833c832574d4ab35c1b606b096, disabling compactions & flushes 2024-11-24T08:30:11,188 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsWrittenViaHRegion,,1732437011008.d677ce833c832574d4ab35c1b606b096. 2024-11-24T08:30:11,188 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsWrittenViaHRegion,,1732437011008.d677ce833c832574d4ab35c1b606b096. 2024-11-24T08:30:11,189 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsWrittenViaHRegion,,1732437011008.d677ce833c832574d4ab35c1b606b096. after waiting 0 ms 2024-11-24T08:30:11,189 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsWrittenViaHRegion,,1732437011008.d677ce833c832574d4ab35c1b606b096. 2024-11-24T08:30:11,190 ERROR [Time-limited test {}] regionserver.HRegion(1960): Memstore data size is 1740 in region testReplayEditsWrittenViaHRegion,,1732437011008.d677ce833c832574d4ab35c1b606b096. 2024-11-24T08:30:11,190 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsWrittenViaHRegion,,1732437011008.d677ce833c832574d4ab35c1b606b096. 2024-11-24T08:30:11,190 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for d677ce833c832574d4ab35c1b606b096: Waiting for close lock at 1732437011188Disabling compacts and flushes for region at 1732437011188Disabling writes for close at 1732437011189 (+1 ms)Writing region close event to WAL at 1732437011189Closed at 1732437011190 (+1 ms) 2024-11-24T08:30:11,192 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34459 is added to blk_1073741927_1106 (size=2805) 2024-11-24T08:30:11,192 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33555 is added to blk_1073741927_1106 (size=2805) 2024-11-24T08:30:11,192 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40533 is added to blk_1073741927_1106 (size=2805) 2024-11-24T08:30:11,209 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:46165/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732437011007/wal.1732437011077, size=2.7 K (2805bytes) 2024-11-24T08:30:11,209 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:46165/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732437011007/wal.1732437011077 2024-11-24T08:30:11,210 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:46165/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732437011007/wal.1732437011077 after 1ms 2024-11-24T08:30:11,212 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:46165/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732437011007/wal.1732437011077: isRecoveredEdits=false, hasTagCompression=true, hasValueCompression=false, valueCompressionType=null 2024-11-24T08:30:11,212 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:46165/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732437011007/wal.1732437011077 took 3ms 2024-11-24T08:30:11,214 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:46165/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732437011007/wal.1732437011077 so closing down 2024-11-24T08:30:11,214 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-24T08:30:11,216 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000003-wal.1732437011077.temp 2024-11-24T08:30:11,217 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:46165/hbase/data/default/testReplayEditsWrittenViaHRegion/d677ce833c832574d4ab35c1b606b096/recovered.edits/0000000000000000003-wal.1732437011077.temp 2024-11-24T08:30:11,217 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-11-24T08:30:11,225 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34459 is added to blk_1073741929_1108 (size=2312) 2024-11-24T08:30:11,225 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33555 is added to blk_1073741929_1108 (size=2312) 2024-11-24T08:30:11,225 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40533 is added to blk_1073741929_1108 (size=2312) 2024-11-24T08:30:11,226 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:46165/hbase/data/default/testReplayEditsWrittenViaHRegion/d677ce833c832574d4ab35c1b606b096/recovered.edits/0000000000000000003-wal.1732437011077.temp (wrote 30 edits, skipped 0 edits in 0 ms) 2024-11-24T08:30:11,227 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:46165/hbase/data/default/testReplayEditsWrittenViaHRegion/d677ce833c832574d4ab35c1b606b096/recovered.edits/0000000000000000003-wal.1732437011077.temp to hdfs://localhost:46165/hbase/data/default/testReplayEditsWrittenViaHRegion/d677ce833c832574d4ab35c1b606b096/recovered.edits/0000000000000000035 2024-11-24T08:30:11,228 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 32 edits across 1 Regions in 15 ms; skipped=2; WAL=hdfs://localhost:46165/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732437011007/wal.1732437011077, size=2.7 K, length=2805, corrupted=false, cancelled=false 2024-11-24T08:30:11,228 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:46165/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732437011007/wal.1732437011077, journal: Splitting hdfs://localhost:46165/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732437011007/wal.1732437011077, size=2.7 K (2805bytes) at 1732437011209Finishing writing output for hdfs://localhost:46165/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732437011007/wal.1732437011077 so closing down at 1732437011214 (+5 ms)Creating recovered edits writer path=hdfs://localhost:46165/hbase/data/default/testReplayEditsWrittenViaHRegion/d677ce833c832574d4ab35c1b606b096/recovered.edits/0000000000000000003-wal.1732437011077.temp at 1732437011217 (+3 ms)3 split writer threads finished at 1732437011217Closed recovered edits writer path=hdfs://localhost:46165/hbase/data/default/testReplayEditsWrittenViaHRegion/d677ce833c832574d4ab35c1b606b096/recovered.edits/0000000000000000003-wal.1732437011077.temp (wrote 30 edits, skipped 0 edits in 0 ms) at 1732437011226 (+9 ms)Rename recovered edits hdfs://localhost:46165/hbase/data/default/testReplayEditsWrittenViaHRegion/d677ce833c832574d4ab35c1b606b096/recovered.edits/0000000000000000003-wal.1732437011077.temp to hdfs://localhost:46165/hbase/data/default/testReplayEditsWrittenViaHRegion/d677ce833c832574d4ab35c1b606b096/recovered.edits/0000000000000000035 at 1732437011227 (+1 ms)Processed 32 edits across 1 Regions in 15 ms; skipped=2; WAL=hdfs://localhost:46165/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732437011007/wal.1732437011077, size=2.7 K, length=2805, corrupted=false, cancelled=false at 1732437011228 (+1 ms) 2024-11-24T08:30:11,229 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:46165/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732437011007/wal.1732437011077 to hdfs://localhost:46165/hbase/oldWALs/wal.1732437011077 2024-11-24T08:30:11,230 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:46165/hbase/data/default/testReplayEditsWrittenViaHRegion/d677ce833c832574d4ab35c1b606b096/recovered.edits/0000000000000000035 2024-11-24T08:30:11,230 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-24T08:30:11,232 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:46165/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732437011007, archiveDir=hdfs://localhost:46165/hbase/oldWALs, maxLogs=32 2024-11-24T08:30:11,246 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732437011007/wal.1732437011232, exclude list is [], retry=0 2024-11-24T08:30:11,248 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40533,DS-5eeb9666-6dd5-4a48-9fb1-3f9aff1acdca,DISK] 2024-11-24T08:30:11,249 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:34459,DS-6b4d1656-f49b-4a16-be49-2eb682c16e11,DISK] 2024-11-24T08:30:11,249 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33555,DS-8d97c3e2-1525-418c-a970-00a7157a0d31,DISK] 2024-11-24T08:30:11,250 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732437011007/wal.1732437011232 2024-11-24T08:30:11,251 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:42673:42673),(127.0.0.1/127.0.0.1:44131:44131),(127.0.0.1/127.0.0.1:39357:39357)] 2024-11-24T08:30:11,251 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => d677ce833c832574d4ab35c1b606b096, NAME => 'testReplayEditsWrittenViaHRegion,,1732437011008.d677ce833c832574d4ab35c1b606b096.', STARTKEY => '', ENDKEY => ''} 2024-11-24T08:30:11,251 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenViaHRegion,,1732437011008.d677ce833c832574d4ab35c1b606b096.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T08:30:11,251 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for d677ce833c832574d4ab35c1b606b096 2024-11-24T08:30:11,251 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for d677ce833c832574d4ab35c1b606b096 2024-11-24T08:30:11,253 INFO [StoreOpener-d677ce833c832574d4ab35c1b606b096-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region d677ce833c832574d4ab35c1b606b096 2024-11-24T08:30:11,254 INFO [StoreOpener-d677ce833c832574d4ab35c1b606b096-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region d677ce833c832574d4ab35c1b606b096 columnFamilyName a 2024-11-24T08:30:11,254 DEBUG [StoreOpener-d677ce833c832574d4ab35c1b606b096-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:30:11,259 DEBUG [StoreOpener-d677ce833c832574d4ab35c1b606b096-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:46165/hbase/data/default/testReplayEditsWrittenViaHRegion/d677ce833c832574d4ab35c1b606b096/a/439792cb36bf4b9ca42dbdd87a1a73d9 2024-11-24T08:30:11,259 INFO [StoreOpener-d677ce833c832574d4ab35c1b606b096-1 {}] regionserver.HStore(327): Store=d677ce833c832574d4ab35c1b606b096/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T08:30:11,259 INFO [StoreOpener-d677ce833c832574d4ab35c1b606b096-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region d677ce833c832574d4ab35c1b606b096 2024-11-24T08:30:11,260 INFO [StoreOpener-d677ce833c832574d4ab35c1b606b096-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region d677ce833c832574d4ab35c1b606b096 columnFamilyName b 2024-11-24T08:30:11,260 DEBUG [StoreOpener-d677ce833c832574d4ab35c1b606b096-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:30:11,260 INFO [StoreOpener-d677ce833c832574d4ab35c1b606b096-1 {}] regionserver.HStore(327): Store=d677ce833c832574d4ab35c1b606b096/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T08:30:11,260 INFO [StoreOpener-d677ce833c832574d4ab35c1b606b096-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region d677ce833c832574d4ab35c1b606b096 2024-11-24T08:30:11,261 INFO [StoreOpener-d677ce833c832574d4ab35c1b606b096-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region d677ce833c832574d4ab35c1b606b096 columnFamilyName c 2024-11-24T08:30:11,261 DEBUG [StoreOpener-d677ce833c832574d4ab35c1b606b096-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:30:11,261 INFO [StoreOpener-d677ce833c832574d4ab35c1b606b096-1 {}] regionserver.HStore(327): Store=d677ce833c832574d4ab35c1b606b096/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T08:30:11,262 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for d677ce833c832574d4ab35c1b606b096 2024-11-24T08:30:11,262 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46165/hbase/data/default/testReplayEditsWrittenViaHRegion/d677ce833c832574d4ab35c1b606b096 2024-11-24T08:30:11,263 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:46165/hbase/data/default/testReplayEditsWrittenViaHRegion/d677ce833c832574d4ab35c1b606b096 2024-11-24T08:30:11,264 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:46165/hbase/data/default/testReplayEditsWrittenViaHRegion/d677ce833c832574d4ab35c1b606b096/recovered.edits/0000000000000000035 2024-11-24T08:30:11,266 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:46165/hbase/data/default/testReplayEditsWrittenViaHRegion/d677ce833c832574d4ab35c1b606b096/recovered.edits/0000000000000000035: isRecoveredEdits=true, hasTagCompression=true, hasValueCompression=false, valueCompressionType=null 2024-11-24T08:30:11,267 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 20, skipped 10, firstSequenceIdInLog=3, maxSequenceIdInLog=35, path=hdfs://localhost:46165/hbase/data/default/testReplayEditsWrittenViaHRegion/d677ce833c832574d4ab35c1b606b096/recovered.edits/0000000000000000035 2024-11-24T08:30:11,267 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing d677ce833c832574d4ab35c1b606b096 3/3 column families, dataSize=1.70 KB heapSize=3.88 KB 2024-11-24T08:30:11,282 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46165/hbase/data/default/testReplayEditsWrittenViaHRegion/d677ce833c832574d4ab35c1b606b096/.tmp/b/609d20aaf6da4ae2994b536fc0f29a98 is 91, key is testReplayEditsWrittenViaHRegion/b:x0/1732437011167/Put/seqid=0 2024-11-24T08:30:11,289 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34459 is added to blk_1073741931_1110 (size=5958) 2024-11-24T08:30:11,289 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33555 is added to blk_1073741931_1110 (size=5958) 2024-11-24T08:30:11,289 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40533 is added to blk_1073741931_1110 (size=5958) 2024-11-24T08:30:11,290 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=35 (bloomFilter=true), to=hdfs://localhost:46165/hbase/data/default/testReplayEditsWrittenViaHRegion/d677ce833c832574d4ab35c1b606b096/.tmp/b/609d20aaf6da4ae2994b536fc0f29a98 2024-11-24T08:30:11,310 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46165/hbase/data/default/testReplayEditsWrittenViaHRegion/d677ce833c832574d4ab35c1b606b096/.tmp/c/60ad63682aa64acebfd271139794af3d is 91, key is testReplayEditsWrittenViaHRegion/c:x0/1732437011176/Put/seqid=0 2024-11-24T08:30:11,316 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34459 is added to blk_1073741932_1111 (size=5958) 2024-11-24T08:30:11,316 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33555 is added to blk_1073741932_1111 (size=5958) 2024-11-24T08:30:11,317 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40533 is added to blk_1073741932_1111 (size=5958) 2024-11-24T08:30:11,317 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=35 (bloomFilter=true), to=hdfs://localhost:46165/hbase/data/default/testReplayEditsWrittenViaHRegion/d677ce833c832574d4ab35c1b606b096/.tmp/c/60ad63682aa64acebfd271139794af3d 2024-11-24T08:30:11,322 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46165/hbase/data/default/testReplayEditsWrittenViaHRegion/d677ce833c832574d4ab35c1b606b096/.tmp/b/609d20aaf6da4ae2994b536fc0f29a98 as hdfs://localhost:46165/hbase/data/default/testReplayEditsWrittenViaHRegion/d677ce833c832574d4ab35c1b606b096/b/609d20aaf6da4ae2994b536fc0f29a98 2024-11-24T08:30:11,327 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46165/hbase/data/default/testReplayEditsWrittenViaHRegion/d677ce833c832574d4ab35c1b606b096/b/609d20aaf6da4ae2994b536fc0f29a98, entries=10, sequenceid=35, filesize=5.8 K 2024-11-24T08:30:11,328 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46165/hbase/data/default/testReplayEditsWrittenViaHRegion/d677ce833c832574d4ab35c1b606b096/.tmp/c/60ad63682aa64acebfd271139794af3d as hdfs://localhost:46165/hbase/data/default/testReplayEditsWrittenViaHRegion/d677ce833c832574d4ab35c1b606b096/c/60ad63682aa64acebfd271139794af3d 2024-11-24T08:30:11,332 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46165/hbase/data/default/testReplayEditsWrittenViaHRegion/d677ce833c832574d4ab35c1b606b096/c/60ad63682aa64acebfd271139794af3d, entries=10, sequenceid=35, filesize=5.8 K 2024-11-24T08:30:11,332 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~1.70 KB/1740, heapSize ~3.59 KB/3680, currentSize=0 B/0 for d677ce833c832574d4ab35c1b606b096 in 65ms, sequenceid=35, compaction requested=false; wal=null 2024-11-24T08:30:11,333 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:46165/hbase/data/default/testReplayEditsWrittenViaHRegion/d677ce833c832574d4ab35c1b606b096/recovered.edits/0000000000000000035 2024-11-24T08:30:11,333 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for d677ce833c832574d4ab35c1b606b096 2024-11-24T08:30:11,333 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for d677ce833c832574d4ab35c1b606b096 2024-11-24T08:30:11,334 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsWrittenViaHRegion descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-24T08:30:11,335 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for d677ce833c832574d4ab35c1b606b096 2024-11-24T08:30:11,337 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:46165/hbase/data/default/testReplayEditsWrittenViaHRegion/d677ce833c832574d4ab35c1b606b096/recovered.edits/35.seqid, newMaxSeqId=35, maxSeqId=1 2024-11-24T08:30:11,338 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened d677ce833c832574d4ab35c1b606b096; next sequenceid=36; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=65390523, jitterRate=-0.025605276226997375}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-24T08:30:11,338 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for d677ce833c832574d4ab35c1b606b096: Writing region info on filesystem at 1732437011251Initializing all the Stores at 1732437011252 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732437011252Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732437011253 (+1 ms)Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732437011253Obtaining lock to block concurrent updates at 1732437011267 (+14 ms)Preparing flush snapshotting stores in d677ce833c832574d4ab35c1b606b096 at 1732437011267Finished memstore snapshotting testReplayEditsWrittenViaHRegion,,1732437011008.d677ce833c832574d4ab35c1b606b096., syncing WAL and waiting on mvcc, flushsize=dataSize=1740, getHeapSize=3920, getOffHeapSize=0, getCellsCount=20 at 1732437011267Flushing stores of testReplayEditsWrittenViaHRegion,,1732437011008.d677ce833c832574d4ab35c1b606b096. at 1732437011267Flushing d677ce833c832574d4ab35c1b606b096/b: creating writer at 1732437011267Flushing d677ce833c832574d4ab35c1b606b096/b: appending metadata at 1732437011281 (+14 ms)Flushing d677ce833c832574d4ab35c1b606b096/b: closing flushed file at 1732437011281Flushing d677ce833c832574d4ab35c1b606b096/c: creating writer at 1732437011295 (+14 ms)Flushing d677ce833c832574d4ab35c1b606b096/c: appending metadata at 1732437011309 (+14 ms)Flushing d677ce833c832574d4ab35c1b606b096/c: closing flushed file at 1732437011309Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@2635f8e1: reopening flushed file at 1732437011321 (+12 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@7b5b6398: reopening flushed file at 1732437011327 (+6 ms)Finished flush of dataSize ~1.70 KB/1740, heapSize ~3.59 KB/3680, currentSize=0 B/0 for d677ce833c832574d4ab35c1b606b096 in 65ms, sequenceid=35, compaction requested=false; wal=null at 1732437011332 (+5 ms)Cleaning up temporary data from old regions at 1732437011333 (+1 ms)Region opened successfully at 1732437011338 (+5 ms) 2024-11-24T08:30:11,405 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:46165/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732437011007/wal.1732437011232, size=0 (0bytes) 2024-11-24T08:30:11,405 WARN [Time-limited test {}] wal.WALSplitter(453): File hdfs://localhost:46165/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732437011007/wal.1732437011232 might be still open, length is 0 2024-11-24T08:30:11,405 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:46165/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732437011007/wal.1732437011232 2024-11-24T08:30:11,406 WARN [IPC Server handler 2 on default port 46165 {}] namenode.FSNamesystem(3854): DIR* NameSystem.internalReleaseLease: File /hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732437011007/wal.1732437011232 has not been closed. Lease recovery is in progress. RecoveryId = 1112 for block blk_1073741930_1109 2024-11-24T08:30:11,406 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=0 on file=hdfs://localhost:46165/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732437011007/wal.1732437011232 after 1ms 2024-11-24T08:30:11,749 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1541013681_22 at /127.0.0.1:48230 [Receiving block BP-1907186785-172.17.0.2-1732436973683:blk_1073741930_1109] {}] datanode.DataXceiver(331): 127.0.0.1:40533:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:48230 dst: /127.0.0.1:40533 java.io.InterruptedIOException: Interrupted while waiting for IO on channel java.nio.channels.SocketChannel[connected local=localhost/127.0.0.1:40533 remote=/127.0.0.1:48230]. Total timeout mills is 60000, 59618 millis timeout left. at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:350) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?] at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?] at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?] at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?] at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?] at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T08:30:11,750 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1541013681_22 at /127.0.0.1:38086 [Receiving block BP-1907186785-172.17.0.2-1732436973683:blk_1073741930_1109] {}] datanode.DataXceiver(331): 127.0.0.1:34459:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:38086 dst: /127.0.0.1:34459 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T08:30:11,750 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1541013681_22 at /127.0.0.1:39102 [Receiving block BP-1907186785-172.17.0.2-1732436973683:blk_1073741930_1109] {}] datanode.DataXceiver(331): 127.0.0.1:33555:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:39102 dst: /127.0.0.1:33555 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T08:30:11,754 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40533 is added to blk_1073741930_1112 (size=2304) 2024-11-24T08:30:11,757 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34459 is added to blk_1073741930_1112 (size=2304) 2024-11-24T08:30:15,407 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=1 on file=hdfs://localhost:46165/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732437011007/wal.1732437011232 after 4002ms 2024-11-24T08:30:15,410 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:46165/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732437011007/wal.1732437011232: isRecoveredEdits=false, hasTagCompression=true, hasValueCompression=false, valueCompressionType=null 2024-11-24T08:30:15,410 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:46165/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732437011007/wal.1732437011232 took 4005ms 2024-11-24T08:30:15,412 INFO [Time-limited test {}] wal.WALSplitter(493): EOF from hdfs://localhost:46165/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732437011007/wal.1732437011232; continuing. 2024-11-24T08:30:15,412 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:46165/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732437011007/wal.1732437011232 so closing down 2024-11-24T08:30:15,412 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-24T08:30:15,413 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000037-wal.1732437011232.temp 2024-11-24T08:30:15,416 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:46165/hbase/data/default/testReplayEditsWrittenViaHRegion/d677ce833c832574d4ab35c1b606b096/recovered.edits/0000000000000000037-wal.1732437011232.temp 2024-11-24T08:30:15,417 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-11-24T08:30:15,422 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40533 is added to blk_1073741933_1113 (size=2312) 2024-11-24T08:30:15,423 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33555 is added to blk_1073741933_1113 (size=2312) 2024-11-24T08:30:15,423 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34459 is added to blk_1073741933_1113 (size=2312) 2024-11-24T08:30:15,423 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:46165/hbase/data/default/testReplayEditsWrittenViaHRegion/d677ce833c832574d4ab35c1b606b096/recovered.edits/0000000000000000037-wal.1732437011232.temp (wrote 30 edits, skipped 0 edits in 0 ms) 2024-11-24T08:30:15,425 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:46165/hbase/data/default/testReplayEditsWrittenViaHRegion/d677ce833c832574d4ab35c1b606b096/recovered.edits/0000000000000000037-wal.1732437011232.temp to hdfs://localhost:46165/hbase/data/default/testReplayEditsWrittenViaHRegion/d677ce833c832574d4ab35c1b606b096/recovered.edits/0000000000000000066 2024-11-24T08:30:15,425 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 30 edits across 1 Regions in 15 ms; skipped=0; WAL=hdfs://localhost:46165/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732437011007/wal.1732437011232, size=0, length=0, corrupted=false, cancelled=false 2024-11-24T08:30:15,425 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:46165/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732437011007/wal.1732437011232, journal: Splitting hdfs://localhost:46165/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732437011007/wal.1732437011232, size=0 (0bytes) at 1732437011405Finishing writing output for hdfs://localhost:46165/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732437011007/wal.1732437011232 so closing down at 1732437015412 (+4007 ms)Creating recovered edits writer path=hdfs://localhost:46165/hbase/data/default/testReplayEditsWrittenViaHRegion/d677ce833c832574d4ab35c1b606b096/recovered.edits/0000000000000000037-wal.1732437011232.temp at 1732437015416 (+4 ms)3 split writer threads finished at 1732437015417 (+1 ms)Closed recovered edits writer path=hdfs://localhost:46165/hbase/data/default/testReplayEditsWrittenViaHRegion/d677ce833c832574d4ab35c1b606b096/recovered.edits/0000000000000000037-wal.1732437011232.temp (wrote 30 edits, skipped 0 edits in 0 ms) at 1732437015423 (+6 ms)Rename recovered edits hdfs://localhost:46165/hbase/data/default/testReplayEditsWrittenViaHRegion/d677ce833c832574d4ab35c1b606b096/recovered.edits/0000000000000000037-wal.1732437011232.temp to hdfs://localhost:46165/hbase/data/default/testReplayEditsWrittenViaHRegion/d677ce833c832574d4ab35c1b606b096/recovered.edits/0000000000000000066 at 1732437015425 (+2 ms)Processed 30 edits across 1 Regions in 15 ms; skipped=0; WAL=hdfs://localhost:46165/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732437011007/wal.1732437011232, size=0, length=0, corrupted=false, cancelled=false at 1732437015425 2024-11-24T08:30:15,426 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:46165/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732437011007/wal.1732437011232 to hdfs://localhost:46165/hbase/oldWALs/wal.1732437011232 2024-11-24T08:30:15,427 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:46165/hbase/data/default/testReplayEditsWrittenViaHRegion/d677ce833c832574d4ab35c1b606b096/recovered.edits/0000000000000000066 2024-11-24T08:30:15,427 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-24T08:30:15,429 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:46165/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732437011007, archiveDir=hdfs://localhost:46165/hbase/oldWALs, maxLogs=32 2024-11-24T08:30:15,443 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732437011007/wal.1732437015429, exclude list is [], retry=0 2024-11-24T08:30:15,446 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40533,DS-5eeb9666-6dd5-4a48-9fb1-3f9aff1acdca,DISK] 2024-11-24T08:30:15,446 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33555,DS-8d97c3e2-1525-418c-a970-00a7157a0d31,DISK] 2024-11-24T08:30:15,446 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:34459,DS-6b4d1656-f49b-4a16-be49-2eb682c16e11,DISK] 2024-11-24T08:30:15,449 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732437011007/wal.1732437015429 2024-11-24T08:30:15,449 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:42673:42673),(127.0.0.1/127.0.0.1:39357:39357),(127.0.0.1/127.0.0.1:44131:44131)] 2024-11-24T08:30:15,449 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenViaHRegion,,1732437011008.d677ce833c832574d4ab35c1b606b096.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T08:30:15,451 INFO [StoreOpener-d677ce833c832574d4ab35c1b606b096-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region d677ce833c832574d4ab35c1b606b096 2024-11-24T08:30:15,452 INFO [StoreOpener-d677ce833c832574d4ab35c1b606b096-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region d677ce833c832574d4ab35c1b606b096 columnFamilyName a 2024-11-24T08:30:15,452 DEBUG [StoreOpener-d677ce833c832574d4ab35c1b606b096-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:30:15,457 DEBUG [StoreOpener-d677ce833c832574d4ab35c1b606b096-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:46165/hbase/data/default/testReplayEditsWrittenViaHRegion/d677ce833c832574d4ab35c1b606b096/a/439792cb36bf4b9ca42dbdd87a1a73d9 2024-11-24T08:30:15,457 INFO [StoreOpener-d677ce833c832574d4ab35c1b606b096-1 {}] regionserver.HStore(327): Store=d677ce833c832574d4ab35c1b606b096/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T08:30:15,457 INFO [StoreOpener-d677ce833c832574d4ab35c1b606b096-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region d677ce833c832574d4ab35c1b606b096 2024-11-24T08:30:15,458 INFO [StoreOpener-d677ce833c832574d4ab35c1b606b096-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region d677ce833c832574d4ab35c1b606b096 columnFamilyName b 2024-11-24T08:30:15,458 DEBUG [StoreOpener-d677ce833c832574d4ab35c1b606b096-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:30:15,463 DEBUG [StoreOpener-d677ce833c832574d4ab35c1b606b096-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:46165/hbase/data/default/testReplayEditsWrittenViaHRegion/d677ce833c832574d4ab35c1b606b096/b/609d20aaf6da4ae2994b536fc0f29a98 2024-11-24T08:30:15,463 INFO [StoreOpener-d677ce833c832574d4ab35c1b606b096-1 {}] regionserver.HStore(327): Store=d677ce833c832574d4ab35c1b606b096/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T08:30:15,463 INFO [StoreOpener-d677ce833c832574d4ab35c1b606b096-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region d677ce833c832574d4ab35c1b606b096 2024-11-24T08:30:15,464 INFO [StoreOpener-d677ce833c832574d4ab35c1b606b096-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region d677ce833c832574d4ab35c1b606b096 columnFamilyName c 2024-11-24T08:30:15,464 DEBUG [StoreOpener-d677ce833c832574d4ab35c1b606b096-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:30:15,469 DEBUG [StoreOpener-d677ce833c832574d4ab35c1b606b096-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:46165/hbase/data/default/testReplayEditsWrittenViaHRegion/d677ce833c832574d4ab35c1b606b096/c/60ad63682aa64acebfd271139794af3d 2024-11-24T08:30:15,469 INFO [StoreOpener-d677ce833c832574d4ab35c1b606b096-1 {}] regionserver.HStore(327): Store=d677ce833c832574d4ab35c1b606b096/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T08:30:15,469 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for d677ce833c832574d4ab35c1b606b096 2024-11-24T08:30:15,470 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46165/hbase/data/default/testReplayEditsWrittenViaHRegion/d677ce833c832574d4ab35c1b606b096 2024-11-24T08:30:15,471 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:46165/hbase/data/default/testReplayEditsWrittenViaHRegion/d677ce833c832574d4ab35c1b606b096 2024-11-24T08:30:15,472 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:46165/hbase/data/default/testReplayEditsWrittenViaHRegion/d677ce833c832574d4ab35c1b606b096/recovered.edits/0000000000000000066 2024-11-24T08:30:15,473 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:46165/hbase/data/default/testReplayEditsWrittenViaHRegion/d677ce833c832574d4ab35c1b606b096/recovered.edits/0000000000000000066: isRecoveredEdits=true, hasTagCompression=true, hasValueCompression=false, valueCompressionType=null 2024-11-24T08:30:15,478 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 30, skipped 0, firstSequenceIdInLog=37, maxSequenceIdInLog=66, path=hdfs://localhost:46165/hbase/data/default/testReplayEditsWrittenViaHRegion/d677ce833c832574d4ab35c1b606b096/recovered.edits/0000000000000000066 2024-11-24T08:30:15,478 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing d677ce833c832574d4ab35c1b606b096 3/3 column families, dataSize=2.55 KB heapSize=5.44 KB 2024-11-24T08:30:15,493 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46165/hbase/data/default/testReplayEditsWrittenViaHRegion/d677ce833c832574d4ab35c1b606b096/.tmp/a/b75cf54d6fca44f98350e613bbfa4505 is 91, key is testReplayEditsWrittenViaHRegion/a:y0/1732437011345/Put/seqid=0 2024-11-24T08:30:15,499 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40533 is added to blk_1073741935_1115 (size=5958) 2024-11-24T08:30:15,499 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33555 is added to blk_1073741935_1115 (size=5958) 2024-11-24T08:30:15,500 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34459 is added to blk_1073741935_1115 (size=5958) 2024-11-24T08:30:15,500 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=66 (bloomFilter=true), to=hdfs://localhost:46165/hbase/data/default/testReplayEditsWrittenViaHRegion/d677ce833c832574d4ab35c1b606b096/.tmp/a/b75cf54d6fca44f98350e613bbfa4505 2024-11-24T08:30:15,518 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46165/hbase/data/default/testReplayEditsWrittenViaHRegion/d677ce833c832574d4ab35c1b606b096/.tmp/b/2d41144bff364b8eb53ff5fa97d18db9 is 91, key is testReplayEditsWrittenViaHRegion/b:y0/1732437011352/Put/seqid=0 2024-11-24T08:30:15,524 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40533 is added to blk_1073741936_1116 (size=5958) 2024-11-24T08:30:15,524 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34459 is added to blk_1073741936_1116 (size=5958) 2024-11-24T08:30:15,524 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33555 is added to blk_1073741936_1116 (size=5958) 2024-11-24T08:30:15,524 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=66 (bloomFilter=true), to=hdfs://localhost:46165/hbase/data/default/testReplayEditsWrittenViaHRegion/d677ce833c832574d4ab35c1b606b096/.tmp/b/2d41144bff364b8eb53ff5fa97d18db9 2024-11-24T08:30:15,543 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46165/hbase/data/default/testReplayEditsWrittenViaHRegion/d677ce833c832574d4ab35c1b606b096/.tmp/c/2c254f7804644db985401d4a6a9756ff is 91, key is testReplayEditsWrittenViaHRegion/c:y0/1732437011360/Put/seqid=0 2024-11-24T08:30:15,550 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40533 is added to blk_1073741937_1117 (size=5958) 2024-11-24T08:30:15,550 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33555 is added to blk_1073741937_1117 (size=5958) 2024-11-24T08:30:15,550 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34459 is added to blk_1073741937_1117 (size=5958) 2024-11-24T08:30:15,551 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=66 (bloomFilter=true), to=hdfs://localhost:46165/hbase/data/default/testReplayEditsWrittenViaHRegion/d677ce833c832574d4ab35c1b606b096/.tmp/c/2c254f7804644db985401d4a6a9756ff 2024-11-24T08:30:15,556 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46165/hbase/data/default/testReplayEditsWrittenViaHRegion/d677ce833c832574d4ab35c1b606b096/.tmp/a/b75cf54d6fca44f98350e613bbfa4505 as hdfs://localhost:46165/hbase/data/default/testReplayEditsWrittenViaHRegion/d677ce833c832574d4ab35c1b606b096/a/b75cf54d6fca44f98350e613bbfa4505 2024-11-24T08:30:15,560 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46165/hbase/data/default/testReplayEditsWrittenViaHRegion/d677ce833c832574d4ab35c1b606b096/a/b75cf54d6fca44f98350e613bbfa4505, entries=10, sequenceid=66, filesize=5.8 K 2024-11-24T08:30:15,561 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46165/hbase/data/default/testReplayEditsWrittenViaHRegion/d677ce833c832574d4ab35c1b606b096/.tmp/b/2d41144bff364b8eb53ff5fa97d18db9 as hdfs://localhost:46165/hbase/data/default/testReplayEditsWrittenViaHRegion/d677ce833c832574d4ab35c1b606b096/b/2d41144bff364b8eb53ff5fa97d18db9 2024-11-24T08:30:15,565 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46165/hbase/data/default/testReplayEditsWrittenViaHRegion/d677ce833c832574d4ab35c1b606b096/b/2d41144bff364b8eb53ff5fa97d18db9, entries=10, sequenceid=66, filesize=5.8 K 2024-11-24T08:30:15,566 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46165/hbase/data/default/testReplayEditsWrittenViaHRegion/d677ce833c832574d4ab35c1b606b096/.tmp/c/2c254f7804644db985401d4a6a9756ff as hdfs://localhost:46165/hbase/data/default/testReplayEditsWrittenViaHRegion/d677ce833c832574d4ab35c1b606b096/c/2c254f7804644db985401d4a6a9756ff 2024-11-24T08:30:15,570 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46165/hbase/data/default/testReplayEditsWrittenViaHRegion/d677ce833c832574d4ab35c1b606b096/c/2c254f7804644db985401d4a6a9756ff, entries=10, sequenceid=66, filesize=5.8 K 2024-11-24T08:30:15,570 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~2.55 KB/2610, heapSize ~5.39 KB/5520, currentSize=0 B/0 for d677ce833c832574d4ab35c1b606b096 in 92ms, sequenceid=66, compaction requested=false; wal=null 2024-11-24T08:30:15,571 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:46165/hbase/data/default/testReplayEditsWrittenViaHRegion/d677ce833c832574d4ab35c1b606b096/recovered.edits/0000000000000000066 2024-11-24T08:30:15,572 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for d677ce833c832574d4ab35c1b606b096 2024-11-24T08:30:15,572 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for d677ce833c832574d4ab35c1b606b096 2024-11-24T08:30:15,572 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsWrittenViaHRegion descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-24T08:30:15,574 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for d677ce833c832574d4ab35c1b606b096 2024-11-24T08:30:15,575 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:46165/hbase/data/default/testReplayEditsWrittenViaHRegion/d677ce833c832574d4ab35c1b606b096/recovered.edits/66.seqid, newMaxSeqId=66, maxSeqId=35 2024-11-24T08:30:15,576 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened d677ce833c832574d4ab35c1b606b096; next sequenceid=67; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=70777388, jitterRate=0.05466526746749878}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-24T08:30:15,576 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for d677ce833c832574d4ab35c1b606b096: Writing region info on filesystem at 1732437015449Initializing all the Stores at 1732437015450 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732437015450Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732437015451 (+1 ms)Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732437015451Obtaining lock to block concurrent updates at 1732437015479 (+28 ms)Preparing flush snapshotting stores in d677ce833c832574d4ab35c1b606b096 at 1732437015479Finished memstore snapshotting testReplayEditsWrittenViaHRegion,,1732437011008.d677ce833c832574d4ab35c1b606b096., syncing WAL and waiting on mvcc, flushsize=dataSize=2610, getHeapSize=5520, getOffHeapSize=0, getCellsCount=30 at 1732437015479Flushing stores of testReplayEditsWrittenViaHRegion,,1732437011008.d677ce833c832574d4ab35c1b606b096. at 1732437015479Flushing d677ce833c832574d4ab35c1b606b096/a: creating writer at 1732437015479Flushing d677ce833c832574d4ab35c1b606b096/a: appending metadata at 1732437015493 (+14 ms)Flushing d677ce833c832574d4ab35c1b606b096/a: closing flushed file at 1732437015493Flushing d677ce833c832574d4ab35c1b606b096/b: creating writer at 1732437015504 (+11 ms)Flushing d677ce833c832574d4ab35c1b606b096/b: appending metadata at 1732437015517 (+13 ms)Flushing d677ce833c832574d4ab35c1b606b096/b: closing flushed file at 1732437015517Flushing d677ce833c832574d4ab35c1b606b096/c: creating writer at 1732437015529 (+12 ms)Flushing d677ce833c832574d4ab35c1b606b096/c: appending metadata at 1732437015542 (+13 ms)Flushing d677ce833c832574d4ab35c1b606b096/c: closing flushed file at 1732437015542Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@8c153aa: reopening flushed file at 1732437015555 (+13 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@4456f877: reopening flushed file at 1732437015560 (+5 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@1bbbfc83: reopening flushed file at 1732437015566 (+6 ms)Finished flush of dataSize ~2.55 KB/2610, heapSize ~5.39 KB/5520, currentSize=0 B/0 for d677ce833c832574d4ab35c1b606b096 in 92ms, sequenceid=66, compaction requested=false; wal=null at 1732437015570 (+4 ms)Cleaning up temporary data from old regions at 1732437015572 (+2 ms)Region opened successfully at 1732437015576 (+4 ms) 2024-11-24T08:30:15,588 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing d677ce833c832574d4ab35c1b606b096, disabling compactions & flushes 2024-11-24T08:30:15,589 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsWrittenViaHRegion,,1732437011008.d677ce833c832574d4ab35c1b606b096. 2024-11-24T08:30:15,589 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsWrittenViaHRegion,,1732437011008.d677ce833c832574d4ab35c1b606b096. 2024-11-24T08:30:15,589 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsWrittenViaHRegion,,1732437011008.d677ce833c832574d4ab35c1b606b096. after waiting 0 ms 2024-11-24T08:30:15,589 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsWrittenViaHRegion,,1732437011008.d677ce833c832574d4ab35c1b606b096. 2024-11-24T08:30:15,590 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsWrittenViaHRegion,,1732437011008.d677ce833c832574d4ab35c1b606b096. 2024-11-24T08:30:15,590 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for d677ce833c832574d4ab35c1b606b096: Waiting for close lock at 1732437015588Disabling compacts and flushes for region at 1732437015588Disabling writes for close at 1732437015589 (+1 ms)Writing region close event to WAL at 1732437015590 (+1 ms)Closed at 1732437015590 2024-11-24T08:30:15,592 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40533 is added to blk_1073741934_1114 (size=93) 2024-11-24T08:30:15,592 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34459 is added to blk_1073741934_1114 (size=93) 2024-11-24T08:30:15,592 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33555 is added to blk_1073741934_1114 (size=93) 2024-11-24T08:30:15,595 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-11-24T08:30:15,595 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL wal:(num 1732437015429) 2024-11-24T08:30:15,612 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestAsyncWALReplayCompressed#testReplayEditsWrittenViaHRegion Thread=444 (was 437) Potentially hanging thread: AsyncFSWAL-28-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-28-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-28-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: LeaseRenewer:jenkinstestReplayEditsWrittenViaHRegion@localhost:46165 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Client (1987708253) connection to localhost/127.0.0.1:46165 from jenkinstestReplayEditsWrittenViaHRegion java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-987159452_22 at /127.0.0.1:48270 [Waiting for operation #18] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-987159452_22 at /127.0.0.1:38126 [Waiting for operation #13] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-987159452_22 at /127.0.0.1:39146 [Waiting for operation #12] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=1412 (was 1344) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=185 (was 202), ProcessCount=11 (was 11), AvailableMemoryMB=6335 (was 6237) - AvailableMemoryMB LEAK? - 2024-11-24T08:30:15,612 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1412 is superior to 1024 2024-11-24T08:30:15,612 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster 2024-11-24T08:30:15,613 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-24T08:30:15,613 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79) at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611) at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-24T08:30:15,613 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T08:30:15,613 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T08:30:15,613 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-24T08:30:15,614 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster 2024-11-24T08:30:15,614 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=1699913015, stopped=false 2024-11-24T08:30:15,614 INFO [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=30c28c82771d,37719,1732436976835 2024-11-24T08:30:15,616 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37719-0x10149184c260000, quorum=127.0.0.1:59068, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-24T08:30:15,616 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37719-0x10149184c260000, quorum=127.0.0.1:59068, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-24T08:30:15,616 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40117-0x10149184c260001, quorum=127.0.0.1:59068, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-24T08:30:15,616 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-24T08:30:15,616 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40241-0x10149184c260002, quorum=127.0.0.1:59068, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-24T08:30:15,616 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40117-0x10149184c260001, quorum=127.0.0.1:59068, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-24T08:30:15,616 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40241-0x10149184c260002, quorum=127.0.0.1:59068, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-24T08:30:15,617 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:37719-0x10149184c260000, quorum=127.0.0.1:59068, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-24T08:30:15,617 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:40117-0x10149184c260001, quorum=127.0.0.1:59068, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-24T08:30:15,617 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-24T08:30:15,618 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277) at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265) at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416) at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-24T08:30:15,618 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T08:30:15,618 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '30c28c82771d,40117,1732436977557' ***** 2024-11-24T08:30:15,618 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:40241-0x10149184c260002, quorum=127.0.0.1:59068, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-24T08:30:15,618 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-11-24T08:30:15,618 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '30c28c82771d,40241,1732436977670' ***** 2024-11-24T08:30:15,618 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-11-24T08:30:15,618 INFO [RS:1;30c28c82771d:40241 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-24T08:30:15,619 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-24T08:30:15,619 INFO [RS:1;30c28c82771d:40241 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-24T08:30:15,619 INFO [RS:1;30c28c82771d:40241 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-24T08:30:15,619 INFO [RS:0;30c28c82771d:40117 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-24T08:30:15,619 INFO [RS:1;30c28c82771d:40241 {}] regionserver.HRegionServer(3091): Received CLOSE for 25de72d74cf42cf025decee5a0d3cd6e 2024-11-24T08:30:15,619 INFO [RS:0;30c28c82771d:40117 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-24T08:30:15,619 INFO [RS:0;30c28c82771d:40117 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-24T08:30:15,619 INFO [RS:0;30c28c82771d:40117 {}] regionserver.HRegionServer(959): stopping server 30c28c82771d,40117,1732436977557 2024-11-24T08:30:15,619 INFO [RS:0;30c28c82771d:40117 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-24T08:30:15,619 INFO [RS:1;30c28c82771d:40241 {}] regionserver.HRegionServer(959): stopping server 30c28c82771d,40241,1732436977670 2024-11-24T08:30:15,619 INFO [RS:0;30c28c82771d:40117 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;30c28c82771d:40117. 2024-11-24T08:30:15,619 INFO [RS:1;30c28c82771d:40241 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-24T08:30:15,619 DEBUG [RS:0;30c28c82771d:40117 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-24T08:30:15,619 INFO [RS:1;30c28c82771d:40241 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:1;30c28c82771d:40241. 2024-11-24T08:30:15,619 DEBUG [RS:0;30c28c82771d:40117 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T08:30:15,619 DEBUG [RS:1;30c28c82771d:40241 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-24T08:30:15,619 INFO [RS:0;30c28c82771d:40117 {}] regionserver.HRegionServer(976): stopping server 30c28c82771d,40117,1732436977557; all regions closed. 2024-11-24T08:30:15,619 DEBUG [RS:1;30c28c82771d:40241 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T08:30:15,619 INFO [RS:1;30c28c82771d:40241 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-24T08:30:15,619 INFO [RS:1;30c28c82771d:40241 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-24T08:30:15,619 INFO [RS:1;30c28c82771d:40241 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-24T08:30:15,619 INFO [RS:1;30c28c82771d:40241 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-11-24T08:30:15,619 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-24T08:30:15,620 DEBUG [RS_CLOSE_REGION-regionserver/30c28c82771d:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 25de72d74cf42cf025decee5a0d3cd6e, disabling compactions & flushes 2024-11-24T08:30:15,620 INFO [RS_CLOSE_REGION-regionserver/30c28c82771d:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testReplayEditsAfterRegionMovedWithMultiCF,,1732436992313.25de72d74cf42cf025decee5a0d3cd6e. 2024-11-24T08:30:15,620 INFO [RS:1;30c28c82771d:40241 {}] regionserver.HRegionServer(1321): Waiting on 2 regions to close 2024-11-24T08:30:15,620 DEBUG [RS_CLOSE_REGION-regionserver/30c28c82771d:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1732436992313.25de72d74cf42cf025decee5a0d3cd6e. 2024-11-24T08:30:15,620 DEBUG [RS:1;30c28c82771d:40241 {}] regionserver.HRegionServer(1325): Online Regions={25de72d74cf42cf025decee5a0d3cd6e=testReplayEditsAfterRegionMovedWithMultiCF,,1732436992313.25de72d74cf42cf025decee5a0d3cd6e., 1588230740=hbase:meta,,1.1588230740} 2024-11-24T08:30:15,620 DEBUG [RS_CLOSE_REGION-regionserver/30c28c82771d:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1732436992313.25de72d74cf42cf025decee5a0d3cd6e. after waiting 0 ms 2024-11-24T08:30:15,620 DEBUG [RS_CLOSE_REGION-regionserver/30c28c82771d:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsAfterRegionMovedWithMultiCF,,1732436992313.25de72d74cf42cf025decee5a0d3cd6e. 2024-11-24T08:30:15,620 DEBUG [RS:1;30c28c82771d:40241 {}] regionserver.HRegionServer(1351): Waiting on 1588230740, 25de72d74cf42cf025decee5a0d3cd6e 2024-11-24T08:30:15,620 DEBUG [RS_CLOSE_META-regionserver/30c28c82771d:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-24T08:30:15,620 INFO [RS_CLOSE_META-regionserver/30c28c82771d:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-24T08:30:15,620 DEBUG [RS_CLOSE_META-regionserver/30c28c82771d:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-24T08:30:15,620 DEBUG [RS_CLOSE_META-regionserver/30c28c82771d:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-24T08:30:15,620 DEBUG [RS_CLOSE_META-regionserver/30c28c82771d:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-24T08:30:15,620 INFO [RS_CLOSE_META-regionserver/30c28c82771d:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=6.86 KB heapSize=11.45 KB 2024-11-24T08:30:15,622 INFO [regionserver/30c28c82771d:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-24T08:30:15,622 INFO [regionserver/30c28c82771d:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-24T08:30:15,622 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33555 is added to blk_1073741835_1011 (size=2054) 2024-11-24T08:30:15,622 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34459 is added to blk_1073741835_1011 (size=2054) 2024-11-24T08:30:15,623 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40533 is added to blk_1073741835_1011 (size=2054) 2024-11-24T08:30:15,625 DEBUG [RS:0;30c28c82771d:40117 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/7507fec3-e300-7585-3a43-9c9d3ed5b09e/oldWALs 2024-11-24T08:30:15,625 INFO [RS:0;30c28c82771d:40117 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 30c28c82771d%2C40117%2C1732436977557:(num 1732436979045) 2024-11-24T08:30:15,625 DEBUG [RS:0;30c28c82771d:40117 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T08:30:15,625 INFO [RS:0;30c28c82771d:40117 {}] regionserver.LeaseManager(133): Closed leases 2024-11-24T08:30:15,625 INFO [RS:0;30c28c82771d:40117 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-24T08:30:15,626 INFO [RS:0;30c28c82771d:40117 {}] hbase.ChoreService(370): Chore service for: regionserver/30c28c82771d:0 had [ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-11-24T08:30:15,626 INFO [RS:0;30c28c82771d:40117 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-24T08:30:15,626 INFO [RS:0;30c28c82771d:40117 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-24T08:30:15,626 INFO [RS:0;30c28c82771d:40117 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-24T08:30:15,626 INFO [RS:0;30c28c82771d:40117 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-24T08:30:15,626 INFO [RS:0;30c28c82771d:40117 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:40117 2024-11-24T08:30:15,627 INFO [regionserver/30c28c82771d:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-24T08:30:15,628 DEBUG [RS_CLOSE_REGION-regionserver/30c28c82771d:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:46165/user/jenkins/test-data/7507fec3-e300-7585-3a43-9c9d3ed5b09e/data/default/testReplayEditsAfterRegionMovedWithMultiCF/25de72d74cf42cf025decee5a0d3cd6e/recovered.edits/20.seqid, newMaxSeqId=20, maxSeqId=17 2024-11-24T08:30:15,628 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37719-0x10149184c260000, quorum=127.0.0.1:59068, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-24T08:30:15,628 INFO [RS:0;30c28c82771d:40117 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-24T08:30:15,629 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40117-0x10149184c260001, quorum=127.0.0.1:59068, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/30c28c82771d,40117,1732436977557 2024-11-24T08:30:15,629 INFO [RS_CLOSE_REGION-regionserver/30c28c82771d:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testReplayEditsAfterRegionMovedWithMultiCF,,1732436992313.25de72d74cf42cf025decee5a0d3cd6e. 2024-11-24T08:30:15,629 DEBUG [RS_CLOSE_REGION-regionserver/30c28c82771d:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 25de72d74cf42cf025decee5a0d3cd6e: Waiting for close lock at 1732437015620Running coprocessor pre-close hooks at 1732437015620Disabling compacts and flushes for region at 1732437015620Disabling writes for close at 1732437015620Writing region close event to WAL at 1732437015623 (+3 ms)Running coprocessor post-close hooks at 1732437015629 (+6 ms)Closed at 1732437015629 2024-11-24T08:30:15,629 DEBUG [RS_CLOSE_REGION-regionserver/30c28c82771d:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testReplayEditsAfterRegionMovedWithMultiCF,,1732436992313.25de72d74cf42cf025decee5a0d3cd6e. 2024-11-24T08:30:15,631 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [30c28c82771d,40117,1732436977557] 2024-11-24T08:30:15,632 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/30c28c82771d,40117,1732436977557 already deleted, retry=false 2024-11-24T08:30:15,632 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; 30c28c82771d,40117,1732436977557 expired; onlineServers=1 2024-11-24T08:30:15,644 DEBUG [RS_CLOSE_META-regionserver/30c28c82771d:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46165/user/jenkins/test-data/7507fec3-e300-7585-3a43-9c9d3ed5b09e/data/hbase/meta/1588230740/.tmp/info/462dd79e7a39461cbc7000006b641cdf is 205, key is testReplayEditsAfterRegionMovedWithMultiCF,,1732436992313.25de72d74cf42cf025decee5a0d3cd6e./info:regioninfo/1732436995420/Put/seqid=0 2024-11-24T08:30:15,651 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40533 is added to blk_1073741938_1118 (size=8243) 2024-11-24T08:30:15,651 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33555 is added to blk_1073741938_1118 (size=8243) 2024-11-24T08:30:15,652 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34459 is added to blk_1073741938_1118 (size=8243) 2024-11-24T08:30:15,652 INFO [RS_CLOSE_META-regionserver/30c28c82771d:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=6.65 KB at sequenceid=21 (bloomFilter=true), to=hdfs://localhost:46165/user/jenkins/test-data/7507fec3-e300-7585-3a43-9c9d3ed5b09e/data/hbase/meta/1588230740/.tmp/info/462dd79e7a39461cbc7000006b641cdf 2024-11-24T08:30:15,673 DEBUG [RS_CLOSE_META-regionserver/30c28c82771d:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46165/user/jenkins/test-data/7507fec3-e300-7585-3a43-9c9d3ed5b09e/data/hbase/meta/1588230740/.tmp/ns/95b37b1ca1c44acdae136f598f2df7d9 is 43, key is default/ns:d/1732436979835/Put/seqid=0 2024-11-24T08:30:15,679 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34459 is added to blk_1073741939_1119 (size=5153) 2024-11-24T08:30:15,680 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40533 is added to blk_1073741939_1119 (size=5153) 2024-11-24T08:30:15,680 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33555 is added to blk_1073741939_1119 (size=5153) 2024-11-24T08:30:15,681 INFO [RS_CLOSE_META-regionserver/30c28c82771d:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=74 B at sequenceid=21 (bloomFilter=true), to=hdfs://localhost:46165/user/jenkins/test-data/7507fec3-e300-7585-3a43-9c9d3ed5b09e/data/hbase/meta/1588230740/.tmp/ns/95b37b1ca1c44acdae136f598f2df7d9 2024-11-24T08:30:15,700 DEBUG [RS_CLOSE_META-regionserver/30c28c82771d:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46165/user/jenkins/test-data/7507fec3-e300-7585-3a43-9c9d3ed5b09e/data/hbase/meta/1588230740/.tmp/table/bed62c670e484f87befc2c0f4fe31944 is 78, key is testReplayEditsAfterRegionMovedWithMultiCF/table:state/1732436992751/Put/seqid=0 2024-11-24T08:30:15,705 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34459 is added to blk_1073741940_1120 (size=5431) 2024-11-24T08:30:15,705 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33555 is added to blk_1073741940_1120 (size=5431) 2024-11-24T08:30:15,706 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40533 is added to blk_1073741940_1120 (size=5431) 2024-11-24T08:30:15,706 INFO [RS_CLOSE_META-regionserver/30c28c82771d:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=148 B at sequenceid=21 (bloomFilter=true), to=hdfs://localhost:46165/user/jenkins/test-data/7507fec3-e300-7585-3a43-9c9d3ed5b09e/data/hbase/meta/1588230740/.tmp/table/bed62c670e484f87befc2c0f4fe31944 2024-11-24T08:30:15,712 DEBUG [RS_CLOSE_META-regionserver/30c28c82771d:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46165/user/jenkins/test-data/7507fec3-e300-7585-3a43-9c9d3ed5b09e/data/hbase/meta/1588230740/.tmp/info/462dd79e7a39461cbc7000006b641cdf as hdfs://localhost:46165/user/jenkins/test-data/7507fec3-e300-7585-3a43-9c9d3ed5b09e/data/hbase/meta/1588230740/info/462dd79e7a39461cbc7000006b641cdf 2024-11-24T08:30:15,717 INFO [RS_CLOSE_META-regionserver/30c28c82771d:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46165/user/jenkins/test-data/7507fec3-e300-7585-3a43-9c9d3ed5b09e/data/hbase/meta/1588230740/info/462dd79e7a39461cbc7000006b641cdf, entries=18, sequenceid=21, filesize=8.0 K 2024-11-24T08:30:15,718 DEBUG [RS_CLOSE_META-regionserver/30c28c82771d:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46165/user/jenkins/test-data/7507fec3-e300-7585-3a43-9c9d3ed5b09e/data/hbase/meta/1588230740/.tmp/ns/95b37b1ca1c44acdae136f598f2df7d9 as hdfs://localhost:46165/user/jenkins/test-data/7507fec3-e300-7585-3a43-9c9d3ed5b09e/data/hbase/meta/1588230740/ns/95b37b1ca1c44acdae136f598f2df7d9 2024-11-24T08:30:15,722 INFO [RS_CLOSE_META-regionserver/30c28c82771d:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46165/user/jenkins/test-data/7507fec3-e300-7585-3a43-9c9d3ed5b09e/data/hbase/meta/1588230740/ns/95b37b1ca1c44acdae136f598f2df7d9, entries=2, sequenceid=21, filesize=5.0 K 2024-11-24T08:30:15,723 DEBUG [RS_CLOSE_META-regionserver/30c28c82771d:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46165/user/jenkins/test-data/7507fec3-e300-7585-3a43-9c9d3ed5b09e/data/hbase/meta/1588230740/.tmp/table/bed62c670e484f87befc2c0f4fe31944 as hdfs://localhost:46165/user/jenkins/test-data/7507fec3-e300-7585-3a43-9c9d3ed5b09e/data/hbase/meta/1588230740/table/bed62c670e484f87befc2c0f4fe31944 2024-11-24T08:30:15,727 INFO [RS_CLOSE_META-regionserver/30c28c82771d:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46165/user/jenkins/test-data/7507fec3-e300-7585-3a43-9c9d3ed5b09e/data/hbase/meta/1588230740/table/bed62c670e484f87befc2c0f4fe31944, entries=2, sequenceid=21, filesize=5.3 K 2024-11-24T08:30:15,728 INFO [RS_CLOSE_META-regionserver/30c28c82771d:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~6.86 KB/7029, heapSize ~11.16 KB/11424, currentSize=0 B/0 for 1588230740 in 108ms, sequenceid=21, compaction requested=false 2024-11-24T08:30:15,731 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40117-0x10149184c260001, quorum=127.0.0.1:59068, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-24T08:30:15,731 INFO [RS:0;30c28c82771d:40117 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-24T08:30:15,731 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40117-0x10149184c260001, quorum=127.0.0.1:59068, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-24T08:30:15,731 INFO [RS:0;30c28c82771d:40117 {}] regionserver.HRegionServer(1031): Exiting; stopping=30c28c82771d,40117,1732436977557; zookeeper connection closed. 2024-11-24T08:30:15,731 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@43c25811 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@43c25811 2024-11-24T08:30:15,732 DEBUG [RS_CLOSE_META-regionserver/30c28c82771d:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:46165/user/jenkins/test-data/7507fec3-e300-7585-3a43-9c9d3ed5b09e/data/hbase/meta/1588230740/recovered.edits/24.seqid, newMaxSeqId=24, maxSeqId=1 2024-11-24T08:30:15,733 DEBUG [RS_CLOSE_META-regionserver/30c28c82771d:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-24T08:30:15,733 INFO [RS_CLOSE_META-regionserver/30c28c82771d:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-24T08:30:15,733 DEBUG [RS_CLOSE_META-regionserver/30c28c82771d:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1732437015620Running coprocessor pre-close hooks at 1732437015620Disabling compacts and flushes for region at 1732437015620Disabling writes for close at 1732437015620Obtaining lock to block concurrent updates at 1732437015620Preparing flush snapshotting stores in 1588230740 at 1732437015620Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=7029, getHeapSize=11664, getOffHeapSize=0, getCellsCount=48 at 1732437015621 (+1 ms)Flushing stores of hbase:meta,,1.1588230740 at 1732437015623 (+2 ms)Flushing 1588230740/info: creating writer at 1732437015623Flushing 1588230740/info: appending metadata at 1732437015644 (+21 ms)Flushing 1588230740/info: closing flushed file at 1732437015644Flushing 1588230740/ns: creating writer at 1732437015658 (+14 ms)Flushing 1588230740/ns: appending metadata at 1732437015673 (+15 ms)Flushing 1588230740/ns: closing flushed file at 1732437015673Flushing 1588230740/table: creating writer at 1732437015686 (+13 ms)Flushing 1588230740/table: appending metadata at 1732437015699 (+13 ms)Flushing 1588230740/table: closing flushed file at 1732437015699Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@fe8c8e8: reopening flushed file at 1732437015711 (+12 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@39a98db: reopening flushed file at 1732437015717 (+6 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@2877bb9: reopening flushed file at 1732437015722 (+5 ms)Finished flush of dataSize ~6.86 KB/7029, heapSize ~11.16 KB/11424, currentSize=0 B/0 for 1588230740 in 108ms, sequenceid=21, compaction requested=false at 1732437015728 (+6 ms)Writing region close event to WAL at 1732437015729 (+1 ms)Running coprocessor post-close hooks at 1732437015733 (+4 ms)Closed at 1732437015733 2024-11-24T08:30:15,733 DEBUG [RS_CLOSE_META-regionserver/30c28c82771d:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-11-24T08:30:15,820 INFO [RS:1;30c28c82771d:40241 {}] regionserver.HRegionServer(976): stopping server 30c28c82771d,40241,1732436977670; all regions closed. 2024-11-24T08:30:15,823 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40533 is added to blk_1073741836_1012 (size=3783) 2024-11-24T08:30:15,823 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34459 is added to blk_1073741836_1012 (size=3783) 2024-11-24T08:30:15,824 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33555 is added to blk_1073741836_1012 (size=3783) 2024-11-24T08:30:15,826 DEBUG [RS:1;30c28c82771d:40241 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/7507fec3-e300-7585-3a43-9c9d3ed5b09e/oldWALs 2024-11-24T08:30:15,826 INFO [RS:1;30c28c82771d:40241 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 30c28c82771d%2C40241%2C1732436977670.meta:.meta(num 1732436979668) 2024-11-24T08:30:15,828 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40533 is added to blk_1073741834_1010 (size=841) 2024-11-24T08:30:15,828 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33555 is added to blk_1073741834_1010 (size=841) 2024-11-24T08:30:15,828 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34459 is added to blk_1073741834_1010 (size=841) 2024-11-24T08:30:15,830 DEBUG [RS:1;30c28c82771d:40241 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/7507fec3-e300-7585-3a43-9c9d3ed5b09e/oldWALs 2024-11-24T08:30:15,830 INFO [RS:1;30c28c82771d:40241 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 30c28c82771d%2C40241%2C1732436977670:(num 1732436979045) 2024-11-24T08:30:15,830 DEBUG [RS:1;30c28c82771d:40241 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T08:30:15,830 INFO [RS:1;30c28c82771d:40241 {}] regionserver.LeaseManager(133): Closed leases 2024-11-24T08:30:15,830 INFO [RS:1;30c28c82771d:40241 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-24T08:30:15,831 INFO [RS:1;30c28c82771d:40241 {}] hbase.ChoreService(370): Chore service for: regionserver/30c28c82771d:0 had [ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS] on shutdown 2024-11-24T08:30:15,831 INFO [RS:1;30c28c82771d:40241 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-24T08:30:15,831 INFO [regionserver/30c28c82771d:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-24T08:30:15,831 INFO [RS:1;30c28c82771d:40241 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:40241 2024-11-24T08:30:15,833 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40241-0x10149184c260002, quorum=127.0.0.1:59068, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/30c28c82771d,40241,1732436977670 2024-11-24T08:30:15,833 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37719-0x10149184c260000, quorum=127.0.0.1:59068, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-24T08:30:15,833 INFO [RS:1;30c28c82771d:40241 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-24T08:30:15,835 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [30c28c82771d,40241,1732436977670] 2024-11-24T08:30:15,839 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/30c28c82771d,40241,1732436977670 already deleted, retry=false 2024-11-24T08:30:15,839 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; 30c28c82771d,40241,1732436977670 expired; onlineServers=0 2024-11-24T08:30:15,839 INFO [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master '30c28c82771d,37719,1732436976835' ***** 2024-11-24T08:30:15,839 INFO [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0 2024-11-24T08:30:15,839 INFO [M:0;30c28c82771d:37719 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-24T08:30:15,840 INFO [M:0;30c28c82771d:37719 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-24T08:30:15,840 DEBUG [M:0;30c28c82771d:37719 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2024-11-24T08:30:15,840 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2024-11-24T08:30:15,840 DEBUG [M:0;30c28c82771d:37719 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2024-11-24T08:30:15,840 DEBUG [master/30c28c82771d:0:becomeActiveMaster-HFileCleaner.large.0-1732436978744 {}] cleaner.HFileCleaner(306): Exit Thread[master/30c28c82771d:0:becomeActiveMaster-HFileCleaner.large.0-1732436978744,5,FailOnTimeoutGroup] 2024-11-24T08:30:15,840 DEBUG [master/30c28c82771d:0:becomeActiveMaster-HFileCleaner.small.0-1732436978746 {}] cleaner.HFileCleaner(306): Exit Thread[master/30c28c82771d:0:becomeActiveMaster-HFileCleaner.small.0-1732436978746,5,FailOnTimeoutGroup] 2024-11-24T08:30:15,840 INFO [M:0;30c28c82771d:37719 {}] hbase.ChoreService(370): Chore service for: master/30c28c82771d:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown 2024-11-24T08:30:15,840 INFO [M:0;30c28c82771d:37719 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-24T08:30:15,840 DEBUG [M:0;30c28c82771d:37719 {}] master.HMaster(1795): Stopping service threads 2024-11-24T08:30:15,840 INFO [M:0;30c28c82771d:37719 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2024-11-24T08:30:15,840 INFO [M:0;30c28c82771d:37719 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-24T08:30:15,841 INFO [M:0;30c28c82771d:37719 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-24T08:30:15,841 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2024-11-24T08:30:15,842 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37719-0x10149184c260000, quorum=127.0.0.1:59068, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2024-11-24T08:30:15,842 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37719-0x10149184c260000, quorum=127.0.0.1:59068, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-24T08:30:15,842 DEBUG [M:0;30c28c82771d:37719 {}] zookeeper.ZKUtil(347): master:37719-0x10149184c260000, quorum=127.0.0.1:59068, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error) 2024-11-24T08:30:15,842 WARN [M:0;30c28c82771d:37719 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null 2024-11-24T08:30:15,843 INFO [M:0;30c28c82771d:37719 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: hdfs://localhost:46165/user/jenkins/test-data/7507fec3-e300-7585-3a43-9c9d3ed5b09e/.lastflushedseqids 2024-11-24T08:30:15,854 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40533 is added to blk_1073741941_1121 (size=138) 2024-11-24T08:30:15,855 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33555 is added to blk_1073741941_1121 (size=138) 2024-11-24T08:30:15,855 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34459 is added to blk_1073741941_1121 (size=138) 2024-11-24T08:30:15,855 INFO [M:0;30c28c82771d:37719 {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-24T08:30:15,856 INFO [M:0;30c28c82771d:37719 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false 2024-11-24T08:30:15,856 DEBUG [M:0;30c28c82771d:37719 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-24T08:30:15,856 INFO [M:0;30c28c82771d:37719 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-24T08:30:15,856 DEBUG [M:0;30c28c82771d:37719 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-24T08:30:15,856 DEBUG [M:0;30c28c82771d:37719 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-24T08:30:15,856 DEBUG [M:0;30c28c82771d:37719 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-24T08:30:15,856 INFO [M:0;30c28c82771d:37719 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=68.33 KB heapSize=83.72 KB 2024-11-24T08:30:15,872 DEBUG [M:0;30c28c82771d:37719 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46165/user/jenkins/test-data/7507fec3-e300-7585-3a43-9c9d3ed5b09e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/9679f85f02cb48ef9f34eeab8d6e63d9 is 82, key is hbase:meta,,1/info:regioninfo/1732436979755/Put/seqid=0 2024-11-24T08:30:15,877 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34459 is added to blk_1073741942_1122 (size=5672) 2024-11-24T08:30:15,877 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33555 is added to blk_1073741942_1122 (size=5672) 2024-11-24T08:30:15,878 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40533 is added to blk_1073741942_1122 (size=5672) 2024-11-24T08:30:15,878 INFO [M:0;30c28c82771d:37719 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=504 B at sequenceid=168 (bloomFilter=true), to=hdfs://localhost:46165/user/jenkins/test-data/7507fec3-e300-7585-3a43-9c9d3ed5b09e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/9679f85f02cb48ef9f34eeab8d6e63d9 2024-11-24T08:30:15,898 DEBUG [M:0;30c28c82771d:37719 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46165/user/jenkins/test-data/7507fec3-e300-7585-3a43-9c9d3ed5b09e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/e7e6a23b9a434f48bd87558c8d5803dc is 1075, key is \x00\x00\x00\x00\x00\x00\x00\x04/proc:d/1732436992757/Put/seqid=0 2024-11-24T08:30:15,904 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33555 is added to blk_1073741943_1123 (size=7754) 2024-11-24T08:30:15,904 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34459 is added to blk_1073741943_1123 (size=7754) 2024-11-24T08:30:15,905 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40533 is added to blk_1073741943_1123 (size=7754) 2024-11-24T08:30:15,905 INFO [M:0;30c28c82771d:37719 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=67.60 KB at sequenceid=168 (bloomFilter=true), to=hdfs://localhost:46165/user/jenkins/test-data/7507fec3-e300-7585-3a43-9c9d3ed5b09e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/e7e6a23b9a434f48bd87558c8d5803dc 2024-11-24T08:30:15,910 INFO [M:0;30c28c82771d:37719 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for e7e6a23b9a434f48bd87558c8d5803dc 2024-11-24T08:30:15,924 DEBUG [M:0;30c28c82771d:37719 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46165/user/jenkins/test-data/7507fec3-e300-7585-3a43-9c9d3ed5b09e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/0d55fd3063ee45e6986e660d4e8f8b1b is 69, key is 30c28c82771d,40117,1732436977557/rs:state/1732436978792/Put/seqid=0 2024-11-24T08:30:15,935 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40241-0x10149184c260002, quorum=127.0.0.1:59068, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-24T08:30:15,935 INFO [RS:1;30c28c82771d:40241 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-24T08:30:15,935 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40241-0x10149184c260002, quorum=127.0.0.1:59068, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-24T08:30:15,935 INFO [RS:1;30c28c82771d:40241 {}] regionserver.HRegionServer(1031): Exiting; stopping=30c28c82771d,40241,1732436977670; zookeeper connection closed. 2024-11-24T08:30:15,935 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@f6f7689 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@f6f7689 2024-11-24T08:30:15,936 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34459 is added to blk_1073741944_1124 (size=5440) 2024-11-24T08:30:15,936 INFO [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 3 regionserver(s) complete 2024-11-24T08:30:15,937 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33555 is added to blk_1073741944_1124 (size=5440) 2024-11-24T08:30:15,937 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40533 is added to blk_1073741944_1124 (size=5440) 2024-11-24T08:30:15,937 INFO [M:0;30c28c82771d:37719 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=249 B at sequenceid=168 (bloomFilter=true), to=hdfs://localhost:46165/user/jenkins/test-data/7507fec3-e300-7585-3a43-9c9d3ed5b09e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/0d55fd3063ee45e6986e660d4e8f8b1b 2024-11-24T08:30:15,942 INFO [M:0;30c28c82771d:37719 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 0d55fd3063ee45e6986e660d4e8f8b1b 2024-11-24T08:30:15,943 DEBUG [M:0;30c28c82771d:37719 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46165/user/jenkins/test-data/7507fec3-e300-7585-3a43-9c9d3ed5b09e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/9679f85f02cb48ef9f34eeab8d6e63d9 as hdfs://localhost:46165/user/jenkins/test-data/7507fec3-e300-7585-3a43-9c9d3ed5b09e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/9679f85f02cb48ef9f34eeab8d6e63d9 2024-11-24T08:30:15,948 INFO [M:0;30c28c82771d:37719 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46165/user/jenkins/test-data/7507fec3-e300-7585-3a43-9c9d3ed5b09e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/9679f85f02cb48ef9f34eeab8d6e63d9, entries=8, sequenceid=168, filesize=5.5 K 2024-11-24T08:30:15,949 DEBUG [M:0;30c28c82771d:37719 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46165/user/jenkins/test-data/7507fec3-e300-7585-3a43-9c9d3ed5b09e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/e7e6a23b9a434f48bd87558c8d5803dc as hdfs://localhost:46165/user/jenkins/test-data/7507fec3-e300-7585-3a43-9c9d3ed5b09e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/e7e6a23b9a434f48bd87558c8d5803dc 2024-11-24T08:30:15,954 INFO [M:0;30c28c82771d:37719 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for e7e6a23b9a434f48bd87558c8d5803dc 2024-11-24T08:30:15,954 INFO [M:0;30c28c82771d:37719 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46165/user/jenkins/test-data/7507fec3-e300-7585-3a43-9c9d3ed5b09e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/e7e6a23b9a434f48bd87558c8d5803dc, entries=17, sequenceid=168, filesize=7.6 K 2024-11-24T08:30:15,955 DEBUG [M:0;30c28c82771d:37719 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46165/user/jenkins/test-data/7507fec3-e300-7585-3a43-9c9d3ed5b09e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/0d55fd3063ee45e6986e660d4e8f8b1b as hdfs://localhost:46165/user/jenkins/test-data/7507fec3-e300-7585-3a43-9c9d3ed5b09e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/0d55fd3063ee45e6986e660d4e8f8b1b 2024-11-24T08:30:15,959 INFO [M:0;30c28c82771d:37719 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 0d55fd3063ee45e6986e660d4e8f8b1b 2024-11-24T08:30:15,959 INFO [M:0;30c28c82771d:37719 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46165/user/jenkins/test-data/7507fec3-e300-7585-3a43-9c9d3ed5b09e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/0d55fd3063ee45e6986e660d4e8f8b1b, entries=3, sequenceid=168, filesize=5.3 K 2024-11-24T08:30:15,960 INFO [M:0;30c28c82771d:37719 {}] regionserver.HRegion(3140): Finished flush of dataSize ~68.33 KB/69972, heapSize ~83.42 KB/85424, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 104ms, sequenceid=168, compaction requested=false 2024-11-24T08:30:15,962 INFO [M:0;30c28c82771d:37719 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-24T08:30:15,962 DEBUG [M:0;30c28c82771d:37719 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1732437015856Disabling compacts and flushes for region at 1732437015856Disabling writes for close at 1732437015856Obtaining lock to block concurrent updates at 1732437015856Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1732437015856Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=69972, getHeapSize=85664, getOffHeapSize=0, getCellsCount=195 at 1732437015856Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1732437015857 (+1 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1732437015857Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1732437015871 (+14 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1732437015871Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1732437015883 (+12 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1732437015897 (+14 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1732437015897Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1732437015910 (+13 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1732437015924 (+14 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1732437015924Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@6f0c6c2e: reopening flushed file at 1732437015943 (+19 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@346e4620: reopening flushed file at 1732437015948 (+5 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@1f9b1254: reopening flushed file at 1732437015954 (+6 ms)Finished flush of dataSize ~68.33 KB/69972, heapSize ~83.42 KB/85424, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 104ms, sequenceid=168, compaction requested=false at 1732437015961 (+7 ms)Writing region close event to WAL at 1732437015962 (+1 ms)Closed at 1732437015962 2024-11-24T08:30:15,964 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34459 is added to blk_1073741830_1006 (size=71325) 2024-11-24T08:30:15,964 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40533 is added to blk_1073741830_1006 (size=71325) 2024-11-24T08:30:15,964 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33555 is added to blk_1073741830_1006 (size=71325) 2024-11-24T08:30:15,965 INFO [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-24T08:30:15,965 INFO [M:0;30c28c82771d:37719 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down. 2024-11-24T08:30:15,965 INFO [M:0;30c28c82771d:37719 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:37719 2024-11-24T08:30:15,966 INFO [M:0;30c28c82771d:37719 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-24T08:30:16,068 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37719-0x10149184c260000, quorum=127.0.0.1:59068, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-24T08:30:16,068 INFO [M:0;30c28c82771d:37719 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-24T08:30:16,068 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37719-0x10149184c260000, quorum=127.0.0.1:59068, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-24T08:30:16,075 ERROR [Time-limited test {}] hdfs.DFSClient(665): Failed to close file: /hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1732436995782/wal.1732436996186 with renewLeaseKey: DEFAULT_16655 java.io.FileNotFoundException: File does not exist: /hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1732436995782/wal.1732436996186 (inode 16655) Holder DFSClient_NONMAPREDUCE_-1541013681_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.ipc.RemoteException.instantiateException(RemoteException.java:121) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.RemoteException.unwrapRemoteException(RemoteException.java:110) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:653) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.close(FanOutOneBlockAsyncDFSOutput.java:619) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hdfs.DummyDFSOutputStream.close(DummyDFSOutputStream.java:52) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeAllFilesBeingWritten(DFSClient.java:662) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeOutputStreams(DFSClient.java:699) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.close(DistributedFileSystem.java:1528) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2194) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2174) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2167) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniDFSCluster(HBaseTestingUtil.java:761) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1021) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.ipc.RemoteException: File does not exist: /hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1732436995782/wal.1732436996186 (inode 16655) Holder DFSClient_NONMAPREDUCE_-1541013681_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1584) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy45.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.lambda$complete$13(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.complete(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:437) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:170) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:162) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:100) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:366) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy46.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:646) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 25 more 2024-11-24T08:30:16,076 ERROR [Time-limited test {}] hdfs.DFSClient(665): Failed to close file: /hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1732436987892/wal.1732436988000 with renewLeaseKey: DEFAULT_16586 java.io.IOException: stream already broken at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.endBlock(FanOutOneBlockAsyncDFSOutput.java:566) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.close(FanOutOneBlockAsyncDFSOutput.java:615) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hdfs.DummyDFSOutputStream.close(DummyDFSOutputStream.java:52) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeAllFilesBeingWritten(DFSClient.java:662) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeOutputStreams(DFSClient.java:699) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.close(DistributedFileSystem.java:1528) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2194) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2174) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2167) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniDFSCluster(HBaseTestingUtil.java:761) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1021) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T08:30:16,076 ERROR [Time-limited test {}] hdfs.DFSClient(665): Failed to close file: /hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732437011007/wal.1732437011232 with renewLeaseKey: DEFAULT_16767 java.io.IOException: stream already broken at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.endBlock(FanOutOneBlockAsyncDFSOutput.java:566) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.close(FanOutOneBlockAsyncDFSOutput.java:615) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hdfs.DummyDFSOutputStream.close(DummyDFSOutputStream.java:52) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeAllFilesBeingWritten(DFSClient.java:662) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeOutputStreams(DFSClient.java:699) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.close(DistributedFileSystem.java:1528) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2194) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2174) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2167) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniDFSCluster(HBaseTestingUtil.java:761) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1021) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T08:30:16,078 ERROR [Time-limited test {}] hdfs.DFSClient(665): Failed to close file: /hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1732436996380/wal.1732437005647 with renewLeaseKey: DEFAULT_16678 java.io.FileNotFoundException: File does not exist: /hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1732436996380/wal.1732437005647 (inode 16678) Holder DFSClient_NONMAPREDUCE_-1541013681_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.ipc.RemoteException.instantiateException(RemoteException.java:121) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.RemoteException.unwrapRemoteException(RemoteException.java:110) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:653) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.close(FanOutOneBlockAsyncDFSOutput.java:619) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hdfs.DummyDFSOutputStream.close(DummyDFSOutputStream.java:52) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeAllFilesBeingWritten(DFSClient.java:662) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeOutputStreams(DFSClient.java:699) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.close(DistributedFileSystem.java:1528) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2194) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2174) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2167) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniDFSCluster(HBaseTestingUtil.java:761) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1021) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.ipc.RemoteException: File does not exist: /hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1732436996380/wal.1732437005647 (inode 16678) Holder DFSClient_NONMAPREDUCE_-1541013681_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1584) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy45.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.lambda$complete$13(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.complete(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:437) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:170) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:162) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:100) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:366) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy46.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:646) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 25 more 2024-11-24T08:30:16,078 ERROR [Time-limited test {}] hdfs.DFSClient(665): Failed to close file: /hbase/WALs/testreplayeditswrittenintowal-manual,16010,1732436981052/wal.1732436981132 with renewLeaseKey: DEFAULT_16506 java.io.IOException: stream already broken at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.endBlock(FanOutOneBlockAsyncDFSOutput.java:566) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.close(FanOutOneBlockAsyncDFSOutput.java:615) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hdfs.DummyDFSOutputStream.close(DummyDFSOutputStream.java:52) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeAllFilesBeingWritten(DFSClient.java:662) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeOutputStreams(DFSClient.java:699) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.close(DistributedFileSystem.java:1528) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2194) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2174) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2167) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniDFSCluster(HBaseTestingUtil.java:761) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1021) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T08:30:16,080 ERROR [Time-limited test {}] hdfs.DFSClient(665): Failed to close file: /hbase/WALs/testdatalosswheninputerror-manual,16010,1732437005862/wal.1732437006370 with renewLeaseKey: DEFAULT_16704 java.io.FileNotFoundException: File does not exist: /hbase/WALs/testdatalosswheninputerror-manual,16010,1732437005862/wal.1732437006370 (inode 16704) Holder DFSClient_NONMAPREDUCE_-1541013681_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.ipc.RemoteException.instantiateException(RemoteException.java:121) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.RemoteException.unwrapRemoteException(RemoteException.java:110) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:653) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.close(FanOutOneBlockAsyncDFSOutput.java:619) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hdfs.DummyDFSOutputStream.close(DummyDFSOutputStream.java:52) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeAllFilesBeingWritten(DFSClient.java:662) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeOutputStreams(DFSClient.java:699) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.close(DistributedFileSystem.java:1528) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2194) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2174) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2167) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniDFSCluster(HBaseTestingUtil.java:761) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1021) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.ipc.RemoteException: File does not exist: /hbase/WALs/testdatalosswheninputerror-manual,16010,1732437005862/wal.1732437006370 (inode 16704) Holder DFSClient_NONMAPREDUCE_-1541013681_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1584) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy45.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.lambda$complete$13(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.complete(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:437) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:170) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:162) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:100) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:366) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy46.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:646) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 25 more 2024-11-24T08:30:16,081 ERROR [Time-limited test {}] hdfs.DFSClient(665): Failed to close file: /hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1732437006529/wal.1732437006608 with renewLeaseKey: DEFAULT_16726 java.io.IOException: stream already broken at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.endBlock(FanOutOneBlockAsyncDFSOutput.java:566) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.close(FanOutOneBlockAsyncDFSOutput.java:615) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hdfs.DummyDFSOutputStream.close(DummyDFSOutputStream.java:52) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeAllFilesBeingWritten(DFSClient.java:662) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeOutputStreams(DFSClient.java:699) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.close(DistributedFileSystem.java:1528) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2194) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2174) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2167) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniDFSCluster(HBaseTestingUtil.java:761) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1021) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T08:30:16,083 ERROR [Time-limited test {}] hdfs.DFSClient(665): Failed to close file: /hbase/WALs/testnameconflictwhensplit1-manual,16010,1732436980691/wal.1732436980902 with renewLeaseKey: DEFAULT_16485 java.io.FileNotFoundException: File does not exist: /hbase/WALs/testnameconflictwhensplit1-manual,16010,1732436980691/wal.1732436980902 (inode 16485) Holder DFSClient_NONMAPREDUCE_-1541013681_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.ipc.RemoteException.instantiateException(RemoteException.java:121) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.RemoteException.unwrapRemoteException(RemoteException.java:110) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:653) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.close(FanOutOneBlockAsyncDFSOutput.java:619) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hdfs.DummyDFSOutputStream.close(DummyDFSOutputStream.java:52) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeAllFilesBeingWritten(DFSClient.java:662) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeOutputStreams(DFSClient.java:699) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.close(DistributedFileSystem.java:1528) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2194) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2174) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2167) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniDFSCluster(HBaseTestingUtil.java:761) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1021) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.ipc.RemoteException: File does not exist: /hbase/WALs/testnameconflictwhensplit1-manual,16010,1732436980691/wal.1732436980902 (inode 16485) Holder DFSClient_NONMAPREDUCE_-1541013681_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1584) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy45.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.lambda$complete$13(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.complete(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:437) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:170) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:162) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:100) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:366) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy46.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:646) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 25 more 2024-11-24T08:30:16,085 ERROR [Time-limited test {}] hdfs.DFSClient(665): Failed to close file: /hbase/WALs/testnameconflictwhensplit0-manual,16010,1732436980167/wal.1732436980421 with renewLeaseKey: DEFAULT_16462 java.io.FileNotFoundException: File does not exist: /hbase/WALs/testnameconflictwhensplit0-manual,16010,1732436980167/wal.1732436980421 (inode 16462) Holder DFSClient_NONMAPREDUCE_-1541013681_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.ipc.RemoteException.instantiateException(RemoteException.java:121) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.RemoteException.unwrapRemoteException(RemoteException.java:110) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:653) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.close(FanOutOneBlockAsyncDFSOutput.java:619) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hdfs.DummyDFSOutputStream.close(DummyDFSOutputStream.java:52) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeAllFilesBeingWritten(DFSClient.java:662) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeOutputStreams(DFSClient.java:699) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.close(DistributedFileSystem.java:1528) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2194) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2174) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2167) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniDFSCluster(HBaseTestingUtil.java:761) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1021) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.ipc.RemoteException: File does not exist: /hbase/WALs/testnameconflictwhensplit0-manual,16010,1732436980167/wal.1732436980421 (inode 16462) Holder DFSClient_NONMAPREDUCE_-1541013681_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1584) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy45.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.lambda$complete$13(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.complete(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:437) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:170) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:162) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:100) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:366) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy46.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:646) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 25 more 2024-11-24T08:30:16,089 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@4290616c{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-24T08:30:16,091 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@5e25d2b6{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-24T08:30:16,091 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-24T08:30:16,091 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@6853f5bc{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-24T08:30:16,092 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@1bf0fdb4{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ed430c83-cb94-7d1a-6ba6-ccf0f0e534d9/hadoop.log.dir/,STOPPED} 2024-11-24T08:30:16,094 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-11-24T08:30:16,094 WARN [BP-1907186785-172.17.0.2-1732436973683 heartbeating to localhost/127.0.0.1:46165 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-11-24T08:30:16,094 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-11-24T08:30:16,094 WARN [BP-1907186785-172.17.0.2-1732436973683 heartbeating to localhost/127.0.0.1:46165 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-1907186785-172.17.0.2-1732436973683 (Datanode Uuid 8a1b4064-c118-49d3-bb11-efcad72fe18b) service to localhost/127.0.0.1:46165 2024-11-24T08:30:16,096 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ed430c83-cb94-7d1a-6ba6-ccf0f0e534d9/cluster_264a055b-3bad-7287-6b90-a8fe44ac1a0a/data/data5/current/BP-1907186785-172.17.0.2-1732436973683 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-24T08:30:16,096 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ed430c83-cb94-7d1a-6ba6-ccf0f0e534d9/cluster_264a055b-3bad-7287-6b90-a8fe44ac1a0a/data/data6/current/BP-1907186785-172.17.0.2-1732436973683 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-24T08:30:16,097 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-11-24T08:30:16,099 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@28e1ba78{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-24T08:30:16,099 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@702c0733{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-24T08:30:16,099 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-24T08:30:16,099 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@3264f7d{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-24T08:30:16,099 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@e971547{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ed430c83-cb94-7d1a-6ba6-ccf0f0e534d9/hadoop.log.dir/,STOPPED} 2024-11-24T08:30:16,100 WARN [BP-1907186785-172.17.0.2-1732436973683 heartbeating to localhost/127.0.0.1:46165 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-11-24T08:30:16,100 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-11-24T08:30:16,100 WARN [BP-1907186785-172.17.0.2-1732436973683 heartbeating to localhost/127.0.0.1:46165 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-1907186785-172.17.0.2-1732436973683 (Datanode Uuid e762698c-7e76-4e21-b6cc-6263e64f5953) service to localhost/127.0.0.1:46165 2024-11-24T08:30:16,100 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-11-24T08:30:16,101 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ed430c83-cb94-7d1a-6ba6-ccf0f0e534d9/cluster_264a055b-3bad-7287-6b90-a8fe44ac1a0a/data/data3/current/BP-1907186785-172.17.0.2-1732436973683 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-24T08:30:16,101 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ed430c83-cb94-7d1a-6ba6-ccf0f0e534d9/cluster_264a055b-3bad-7287-6b90-a8fe44ac1a0a/data/data4/current/BP-1907186785-172.17.0.2-1732436973683 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-24T08:30:16,101 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-11-24T08:30:16,103 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@4dc44ec6{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-24T08:30:16,103 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@17f6e478{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-24T08:30:16,103 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-24T08:30:16,104 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@3d33512e{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-24T08:30:16,104 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@4b895d95{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ed430c83-cb94-7d1a-6ba6-ccf0f0e534d9/hadoop.log.dir/,STOPPED} 2024-11-24T08:30:16,105 WARN [BP-1907186785-172.17.0.2-1732436973683 heartbeating to localhost/127.0.0.1:46165 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-11-24T08:30:16,105 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-11-24T08:30:16,105 WARN [BP-1907186785-172.17.0.2-1732436973683 heartbeating to localhost/127.0.0.1:46165 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-1907186785-172.17.0.2-1732436973683 (Datanode Uuid 3237fd78-a6e4-4c92-99cc-6bca7ba6e9c2) service to localhost/127.0.0.1:46165 2024-11-24T08:30:16,105 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-11-24T08:30:16,106 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ed430c83-cb94-7d1a-6ba6-ccf0f0e534d9/cluster_264a055b-3bad-7287-6b90-a8fe44ac1a0a/data/data1/current/BP-1907186785-172.17.0.2-1732436973683 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-24T08:30:16,106 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ed430c83-cb94-7d1a-6ba6-ccf0f0e534d9/cluster_264a055b-3bad-7287-6b90-a8fe44ac1a0a/data/data2/current/BP-1907186785-172.17.0.2-1732436973683 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-24T08:30:16,106 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-11-24T08:30:16,112 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@1843526c{hdfs,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-11-24T08:30:16,113 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@776c4d96{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-24T08:30:16,113 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-24T08:30:16,113 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@70596a48{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-24T08:30:16,113 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@6630be9c{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ed430c83-cb94-7d1a-6ba6-ccf0f0e534d9/hadoop.log.dir/,STOPPED} 2024-11-24T08:30:16,123 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers 2024-11-24T08:30:16,179 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down